Revert "[BugFix] Fix 'GPUARWorker' attribute error (#5478)"

This reverts commit f4640d9a2e.

.buildkite/amd/test-amd-merge.yml

@@ -89,7 +89,7 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
 
     - label: "Diffusion Model Test"
       agent_pool: mi325_1

.buildkite/amd/test-amd-ready.yml

@@ -35,7 +35,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
 
 
 - label: "Voxtral TTS CUDA Unit Test"

.buildkite/cuda/test-merge.yml

@@ -8,24 +8,14 @@ steps:
   - group: ":card_index_dividers: Simple Test"
     depends_on: upload-merge-pipeline
     steps:
-      - label: "Simple · Model Executor Test"
+      - label: "Simple · Diffusion & Model Executor Test"
         commands:
-          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Diffusion Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Engine&Entrypoints Test"
-        commands:
-          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
-          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Diffusion Test"
@@ -37,6 +27,12 @@ steps:
           - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
         mirror_hardwares: l4_1
 
+      - label: "Diffusion · Offloader Test"
+        timeout_in_minutes: 40
+        commands:
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
+        mirror_hardwares: l4_1
+
   - label: "Distributed Test"
     depends_on: upload-merge-pipeline
     timeout_in_minutes: 60
@@ -139,32 +135,39 @@ steps:
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        mirror_hardwares: h100_2
+        agents:
-
+          queue: "mithril-h100-pool"
-      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        plugins:
-        source_file_dependencies:
+          - kubernetes:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
+              podSpec:
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+                containers:
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+                    resources:
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+                      limits:
-          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+                        nvidia.com/gpu: 2
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+                    volumeMounts:
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+                      - name: devshm
-          - tests/helpers/minicpmo_4_5_duplex.py
+                        mountPath: /dev/shm
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+                      - name: hf-cache
-          - vllm_omni/experimental/fullduplex/
+                        mountPath: /root/.cache/huggingface
-          - vllm_omni/model_executor/models/minicpmo_4_5/
+                    env:
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+                      - name: HF_HOME
-          - vllm_omni/core/sched/
+                        value: /root/.cache/huggingface
-          - vllm_omni/outputs/multimodal_accumulation.py
+                      - name: HF_TOKEN
-          - vllm_omni/engine/stage_init_utils.py
+                        valueFrom:
-          - vllm_omni/engine/async_omni_engine.py
+                          secretKeyRef:
-          - vllm_omni/entrypoints/cli/serve.py
+                            name: hf-token-secret
-        timeout_in_minutes: 50
+                            key: token
-        commands:
+                nodeSelector:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
-        mirror_hardwares: h100_2
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -182,6 +185,22 @@ steps:
             "
         mirror_hardwares: l4_1
 
+      - label: "TTS · SoulX-Singer Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_soulxsinger.py
+          - vllm_omni/diffusion/models/soulx_singer/
+          - vllm_omni/deploy/soulxsinger_svc.yaml
+          - vllm_omni/deploy/soulxsinger_svs.yaml
+        timeout_in_minutes: 30
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
       - label: "Diffusion · Qwen Image Edit Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_qwen_image_edit.py

.buildkite/cuda/test-nightly.yml

@@ -15,13 +15,7 @@ steps:
       - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
-        mirror_hardwares: h100_2
-
-      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
-        timeout_in_minutes: 50
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Omni · Doc Test with L4"
@@ -427,12 +421,6 @@ steps:
           - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test"
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
-
       - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
         timeout_in_minutes: 180
         commands:
@@ -460,63 +448,37 @@ steps:
             exit $$EXIT2
         mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
-        key: nightly-diffusion-x2v-performance-minimax-h3
-        timeout_in_minutes: 180
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-        mirror_hardwares: h100_4
-
   - group: ":card_index_dividers: Diffusion Test"
-    key: nightly-diffusion-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
     steps:
-      - label: ":full_moon: Diffusion · Distributed Test with L4"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
-        mirror_hardwares: l4_4
-
-      - label: ":full_moon: Diffusion · Distributed Test with H100"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_4
 
       - label: ":full_moon: Diffusion · Offloader Test"
-        timeout_in_minutes: 60
+        timeout_in_minutes: 40
         commands:
-          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda"
         mirror_hardwares: l4_1
 
-      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+      - label: ":full_moon: Diffusion · Quantization Test with H100"
-        timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+          - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_2
-
       - label: ":full_moon: Diffusion · Quantization Test with L4"
         timeout_in_minutes: 60
         commands:
-          - pip install "vllm-gguf-plugin>=0.0.3"
           - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
         mirror_hardwares: l4_1
 
+      - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
   - label: ":bar_chart: Testcase Statistics"
     key: nightly-testcase-statistics
     timeout_in_minutes: 120

.buildkite/cuda/test-ready.yml

@@ -8,26 +8,22 @@ steps:
   - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
     steps:
-      - label: "Simple · Model Executor Test"
+      - label: "Simple · Diffusion & Model Executor Test"
         commands:
-          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Diffusion Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Engine&Entrypoints Test"
-        commands:
-          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
-          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
         mirror_hardwares: l4_1
 
+  - label: "Custom Pipeline Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
+    mirror_hardwares: l4_1
+
   - group: ":card_index_dividers: Diffusion Test"
     depends_on: upload-ready-pipeline
     steps:
@@ -108,34 +104,11 @@ steps:
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-ready-pipeline
     steps:
-      - label: "Custom Pipeline Test"
-        source_file_dependencies:
-          - tests/e2e/features/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
-        commands:
-          - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
 
       - label: "RLHF · VeRL-Omni E2E Test"
         source_file_dependencies:
-          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/features/helpers/verl_omni_server.py
+          - tests/e2e/offline_inference/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
           - vllm_omni/diffusion/
           - vllm_omni/engine/
           - vllm_omni/entrypoints/
@@ -144,7 +117,7 @@ steps:
         commands:
           - |
             timeout 20m bash -c "
-              pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+              pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
             "
         mirror_hardwares: l4_1
 
@@ -170,7 +143,34 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
             "
-        mirror_hardwares: h100_2
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
 
       - label: "Omni · MiniCPM-o 4.5 Online Test"
         source_file_dependencies:
@@ -181,34 +181,34 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
             "
-        mirror_hardwares: h100_2
+        agents:
-
+          queue: "mithril-h100-pool"
-      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        plugins:
-        source_file_dependencies:
+          - kubernetes:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
+              podSpec:
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+                containers:
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+                    resources:
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+                      limits:
-          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+                        nvidia.com/gpu: 2
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+                    volumeMounts:
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+                      - name: devshm
-          - tests/helpers/minicpmo_4_5_duplex.py
+                        mountPath: /dev/shm
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+                      - name: hf-cache
-          - vllm_omni/experimental/fullduplex/
+                        mountPath: /root/.cache/huggingface
-          - vllm_omni/model_executor/models/minicpmo_4_5/
+                    env:
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+                      - name: HF_HOME
-          - vllm_omni/core/sched/
+                        value: /root/.cache/huggingface
-          - vllm_omni/outputs/multimodal_accumulation.py
+                nodeSelector:
-          - vllm_omni/engine/stage_init_utils.py
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
-          - vllm_omni/engine/async_omni_engine.py
+                volumes:
-          - vllm_omni/entrypoints/cli/serve.py
+                  - name: devshm
-        commands:
+                    emptyDir:
-          - |
+                      medium: Memory
-            timeout 20m bash -c "
+                  - name: hf-cache
-              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+                    hostPath:
-            "
+                      path: /mnt/hf-cache
-        mirror_hardwares: h100_2
+                      type: DirectoryOrCreate
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -344,3 +344,13 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · GGUF Plugin Test"
+        soft_fail: true
+        source_file_dependencies:
+          - tests/plugin/test_gguf_diffusion.py
+          - vllm_omni/diffusion/model_loader
+        commands:
+          - pip install "vllm-gguf-plugin>=0.0.3"
+          - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
+        mirror_hardwares: l4_1

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.26.0"
+          VLLM_VERSION: "v0.25.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/npu/test-npu-nightly.yml

@@ -95,80 +95,6 @@ steps:
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
             exit $$EXIT
-  - group: ":card_index_dividers: Diffusion I2I Model Test"
-    key: nightly-HunyuanImage3-x2i-test-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
-      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
-          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
-  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
-    key: nightly-diffusion-benchmark-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
   - group: ":card_index_dividers: Diffusion X2V Model Test"
     key: nightly-diffusion-x2v-group
     depends_on: upload-nightly-pipeline
@@ -177,14 +103,6 @@ steps:
       build.pull_request.labels includes "nightly-test" ||
       build.pull_request.labels includes "diffusion-x2v-test"
     steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
-        mirror_hardwares: a2b3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
       - label: ":full_moon: Diffusion X2V · Perf Test"
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180

.claude/skills/add-diffusion-model/SKILL.md

@@ -309,7 +309,7 @@ For Omni or custom models, create:
 
 Required updates:
 1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
-2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
+2. `docs/user_guide/diffusion/cpu_offload_diffusion.md` — if CPU offload supported (add to supported models table)
 3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
 4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
 5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)

.github/CODEOWNERS

@@ -16,15 +16,15 @@
 /.claude/ @hsliuustc0106
 
 # Tests
-/tests/ @yenuo26 @NickCao
+/tests/ @yenuo26
 
 # CI
-/.buildkite/ @yenuo26 @congw729 @NickCao
+/.buildkite/ @yenuo26 @congw729
 
 # Engine and model_executor
 /vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307
 /vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan @Sy0307
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks @NickCao
+/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks
 /vllm_omni/model_executor/ @tzhouam @gcanlin
 /vllm_omni/worker/ @tzhouam @gcanlin
 
@@ -37,7 +37,7 @@
 
 # Omni and TTS support
 /vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin @NickCao
+/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin
 
 # Benchmarks
 /benchmarks/accuracy/ @david6666666

README.md

@@ -16,10 +16,14 @@ Easy, fast, and cheap omni-modality model serving for everyone
 ---
 
 *Latest News* 🔥
-- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
-- [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.
+- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with [VeRL-Omni](https://github.com/verl-project/verl-omni).
+- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
+- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
+- [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
+- [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
+- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---
@@ -51,13 +55,12 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
-- Full-duplex realtime serving with streaming audio input and output (experimental)
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
+- **Omni-modality models** (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
 - **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
+- **Diffusion models** — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
 - **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
 
 ## Getting Started

apps/ComfyUI-vLLM-Omni/README.md

@@ -47,7 +47,7 @@ If no, check your shell running the ComfyUI process. There may be some error mes
 This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):
 
 - **Generate Image** for text-to-image and image-to-image tasks
-- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Generate Video** for text-to-video and image-to-video tasks
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks
 
@@ -105,7 +105,7 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.
 
-### Text-to-video and image-to-video generation (e.g., Wan, MiniMax-H3)
+### Text-to-video and image-to-video generation (e.g., Wan)
 
 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Video Generation**)
 
@@ -117,11 +117,7 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>
 
 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
+> The node automatically choose text-to-video or image-to-video API endpoints depending on whether you connect an image input or not.
->
-> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
->
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
 
 ### TTS (e.g., Qwen TTS series)
 

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -15,13 +15,11 @@ from .comfyui_vllm_omni.nodes import (
     VLLMOmniDiffusionSampling,
     VLLMOmniGenerateImage,
     VLLMOmniGenerateVideo,
-    VLLMOmniMiniMaxH3Params,
     VLLMOmniQwenTTSParams,
     VLLMOmniRemoteLoRA,
     VLLMOmniSamplingParamsList,
     VLLMOmniTTS,
     VLLMOmniUnderstanding,
-    VLLMOmniVideoReferences,
     VLLMOmniVoiceClone,
     VLLMOmniWanParams,
 )
@@ -34,7 +32,6 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
     "VLLMOmniTTS": VLLMOmniTTS,
     "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
-    "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
     # === Params ===
     "VLLMOmniARSampling": VLLMOmniARSampling,
     "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -42,7 +39,6 @@ NODE_CLASS_MAPPINGS = {
     "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
     "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
     "VLLMOmniWanParams": VLLMOmniWanParams,
-    "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
 }
 
 # A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -53,7 +49,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniUnderstanding": "Multimodality Understanding",
     "VLLMOmniTTS": "TTS (Text to Speech)",
     "VLLMOmniVoiceClone": "TTS Voice Cloning",
-    "VLLMOmniVideoReferences": "Video References",
     # === Params ===
     "VLLMOmniARSampling": "AR Sampling Params",
     "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
@@ -61,7 +56,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "VLLMOmniRemoteLoRA": "LoRA",
     "VLLMOmniQwenTTSParams": "Qwen TTS Params",
     "VLLMOmniWanParams": "Wan Video Params",
-    "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",
 }
 
 WEB_DIRECTORY = "./web"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -10,9 +10,7 @@ from .utils.types import (
     AudioFormat,
     AutoregressionSamplingParams,
     DiffusionSamplingParams,
-    MiniMaxH3ModelSpecificParams,
     QwenTTSModelSpecificParams,
-    VideoReferences,
     WanModelSpecificParams,
 )
 from .utils.validators import (
@@ -165,8 +163,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                 "num_frames": ("INT", {"default": 41, "min": 1}),
             },
             "optional": {
-                "frame": ("IMAGE",),
+                "image": ("IMAGE",),
-                "references": ("VIDEO_REFERENCES",),
                 "sampling_params": ("SAMPLING_PARAMS",),
                 "lora": ("REMOTE_LORA",),
                 "model_params": ("VIDEO_PARAMS",),
@@ -177,15 +174,6 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
     RETURN_NAMES = ("video",)
     FUNCTION = "generate"
 
-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
-        base = super().VALIDATE_INPUTS(url, model)
-        if base is not True:
-            return base
-        if frame is not None and references is not None:
-            return "Provide only one of frame or references, not both."
-        return True
-
     async def generate(
         self,
         url: str,
@@ -196,8 +184,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
         fps: int,
         num_frames: int,
         negative_prompt: str | None = None,
-        frame: torch.Tensor | None = None,
+        image: torch.Tensor | None = None,
-        references: dict | None = None,
         sampling_params: dict | list[dict] | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
@@ -220,16 +207,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
 
         if sampling_params is not None:
             sampling_params.pop("type", None)  # internal fields
-        # model_params["type"] is also an internal field, but is used in api_client
+        if model_params is not None:
-        # if model_params is not None:
+            model_params.pop("type", None)  # internal fields
-        #     model_params.pop("type", None)  # internal fields
 
         client = VLLMOmniClient(url)
         output = await client.generate_video(
             model=model,
             prompt=prompt,
-            frame=frame,  # frame present => fl2va / Wan I2V
+            image=image,  # image present => i2v, absent => t2v
-            references=references,
             width=width,
             height=height,
             num_frames=num_frames,
@@ -749,78 +734,3 @@ class VLLMOmniWanParams:
 
     def get_params(self, **kwargs):
         return (WanModelSpecificParams(kwargs),)
-
-
-class VLLMOmniMiniMaxH3Params:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "audio_flow_shift": (
-                    "FLOAT",
-                    {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1},
-                ),
-                "flow_shift": (
-                    "FLOAT",
-                    {"default": 12.0, "min": 0.0, "max": 100.0, "step": 0.1},
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("VIDEO_PARAMS",)
-    RETURN_NAMES = ("MiniMax-H3 video params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Video Params"
-
-    def get_params(self, **kwargs):
-        params = MiniMaxH3ModelSpecificParams(kwargs)
-        params["type"] = "minimax_h3"
-        return (params,)
-
-
-class VLLMOmniVideoReferences:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {},
-            "optional": {
-                "image_1": ("IMAGE",),
-                "image_2": ("IMAGE",),
-                "audio_1": ("AUDIO",),
-                "audio_2": ("AUDIO",),
-                "video_1": ("VIDEO",),
-                "video_2": ("VIDEO",),
-            },
-        }
-
-    RETURN_TYPES = ("VIDEO_REFERENCES",)
-    RETURN_NAMES = ("references",)
-    FUNCTION = "get_references"
-    CATEGORY = "vLLM-Omni"
-
-    def get_references(
-        self,
-        image_1: torch.Tensor | None = None,
-        image_2: torch.Tensor | None = None,
-        audio_1: AudioInput | None = None,
-        audio_2: AudioInput | None = None,
-        video_1: VideoInput | None = None,
-        video_2: VideoInput | None = None,
-        **kwargs,
-    ):
-        if kwargs:
-            logger.info("Uncaught kwargs: %s", kwargs)
-        refs = VideoReferences()
-        if image_1 is not None:
-            refs["image_1"] = image_1
-        if image_2 is not None:
-            refs["image_2"] = image_2
-        if audio_1 is not None:
-            refs["audio_1"] = audio_1
-        if audio_2 is not None:
-            refs["audio_2"] = audio_2
-        if video_1 is not None:
-            refs["video_1"] = video_1
-        if video_2 is not None:
-            refs["video_2"] = video_2
-        return (refs,)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -24,7 +24,6 @@ from .format import (
     image_tensor_to_base64,
     image_tensor_to_png_bytes,
     video_to_base64,
-    video_to_bytes,
 )
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
@@ -64,11 +63,7 @@ async def url_bytes(session: aiohttp.ClientSession, url: str, verb: str = "get",
 
 class VLLMOmniClient:
     def __init__(
-        self,
+        self, base_url: str, timeout: float | None = None, poll_interval: float = 5.0, max_poll_duration: float = 60 * 5
-        base_url: str,
-        timeout: float | None = None,
-        poll_interval: float = 5.0,
-        max_poll_duration: float = 60 * 30,
     ):
         self.base_url = base_url
         self.timeout = aiohttp.ClientTimeout(total=timeout)
@@ -266,17 +261,12 @@ class VLLMOmniClient:
         num_frames: int,
         fps: int,
         negative_prompt: str | None = None,
-        frame: torch.Tensor | None = None,
+        image: torch.Tensor | None = None,
-        references: dict | None = None,
         sampling_params: dict | None = None,
         model_params: dict | None = None,
         lora: dict | None = None,
-        **extra_params,
+        **extra_body,
     ) -> VideoInput:
-        if frame is not None and references is not None:
-            raise ValueError("Provide only one of frame or references, not both.")
-
-        # === regular payload fields ===
         form = aiohttp.FormData()
         form.add_field("model", model)
         form.add_field("prompt", prompt)
@@ -289,91 +279,23 @@ class VLLMOmniClient:
         if sampling_params is not None:
             for k, v in sampling_params.items():
                 form.add_field(k, str(v))
+        if model_params is not None:
+            for k, v in model_params.items():
+                form.add_field(k, str(v))
         if lora is not None:
             form.add_field("lora", json.dumps(lora, ensure_ascii=False))
+        if extra_body:
+            form.add_field("extra_body", json.dumps(extra_body, ensure_ascii=False))
 
-        # === multimodal inputs (first-last-frames, references, etc.) ===
+        if image is not None:
-        input_reference_image: torch.Tensor | None = None
-        audio_reference: AudioInput | None = None
-        reference_videos: list[VideoInput] = []
-        video_task: str | None = None
-
-        if frame is not None:
-            input_reference_image = frame
-            video_task = "fl2va"
-        elif references is not None:
-            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
-            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
-            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
-
-            if not images and not audios and not videos:
-                raise ValueError("references is empty; connect at least one image/audio/video.")
-
-            if videos:
-                if images or audios:
-                    raise ValueError(
-                        "Video references cannot be combined with image or audio references. "
-                        "Connect only video_1/video_2 for multi-video Ref2VA."
-                    )
-                reference_videos = videos
-                video_task = "ref2va"
-            elif len(images) == 1 and len(audios) == 1:
-                input_reference_image = images[0]
-                audio_reference = audios[0]
-                video_task = "ref2va"
-            else:
-                raise ValueError(
-                    "Invalid references combination. Supported modes: "
-                    "(1) one or more videos only, or (2) exactly one image and one audio."
-                )
-        else:
-            video_task = "t2va"
-
-        if input_reference_image is not None:
             image_filename = "image.png"  # Required for multipart form
             form.add_field(
                 "input_reference",
-                image_tensor_to_png_bytes(input_reference_image, image_filename),
+                image_tensor_to_png_bytes(image, image_filename),
                 filename=image_filename,
                 content_type="image/png",
             )
 
-        if audio_reference is not None:
-            form.add_field(
-                "audio_reference",
-                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
-            )
-
-        for idx, video in enumerate(reference_videos, start=1):
-            video_filename = f"reference_{idx}.mp4"
-            form.add_field(
-                "input_references",
-                video_to_bytes(video, video_filename),
-                filename=video_filename,
-                content_type="video/mp4",
-            )
-
-        # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
-        if model_params is not None:
-            model_params = dict(model_params)
-            model_params.pop("type", None)
-
-        spec, _ = lookup_model_spec(model)
-        params_builder = spec.get("params_builder") if spec else None
-        if params_builder is not None:
-            form_fields = params_builder(
-                model_params or {},
-                extra_params={**extra_params, "task": video_task},
-            )
-            for k, v in form_fields.items():
-                form.add_field(k, v if isinstance(v, str) else str(v))
-        else:
-            if model_params is not None:
-                for k, v in model_params.items():
-                    form.add_field(k, str(v))
-            if extra_params:
-                form.add_field("extra_params", json.dumps(extra_params, ensure_ascii=False))
-
         async with aiohttp.ClientSession(timeout=self.timeout) as session:
             # Start the video generation job
             url = f"{self.base_url}/videos"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,12 +1,7 @@
-import json
 import re
-from typing import Any
 
-from .logger import get_logger
 from .types import Modality, ModelMode, Spec
 
-logger = get_logger(__name__)
-
 
 def _bagel_payload_preprocessor(payload: dict) -> dict:
     try:
@@ -34,30 +29,6 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
     return payload
 
 
-def _minimaxh3_params_builder(
-    model_params: dict[str, Any],
-    *,
-    extra_params: dict[str, Any],
-) -> dict[str, Any]:
-    """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
-    params = dict(model_params)
-    params.pop("type", None)
-    form_fields: dict[str, Any] = {}
-    merged_extra_params: dict[str, Any] = dict(extra_params or {})
-
-    if "flow_shift" in params:
-        form_fields["flow_shift"] = params.pop("flow_shift")
-    for key in ("audio_flow_shift",):
-        if key in params:
-            merged_extra_params[key] = params.pop(key)
-    if params:
-        logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
-
-    if merged_extra_params:
-        form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
-    return form_fields
-
-
 _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
     r"BAGEL-7B-MoT": {
         "stages": [
@@ -100,21 +71,6 @@ _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
             }
         ],
     },
-    r"MiniMax-H3(/FL2VA|/Ref2VA)?": {
-        "stages": ["diffusion"],
-        "modes": [
-            {
-                "mode": ModelMode.VIDEO_GENERATION,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-        "params_builder": _minimaxh3_params_builder,
-    },
 }
 # Convert dict keys to regex patterns
 MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
@@ -124,10 +80,12 @@ del _MODEL_PIPELINE_SPECS
 
 
 def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
-    normalized = model.rstrip("/").lstrip("/")
+    try:
-    lookup_key = normalized.split("/", 1)[-1]
+        last_component = model.rstrip("/").rsplit("/", 1)[-1]
+    except IndexError:
+        last_component = model
     for pattern, spec in MODEL_PIPELINE_SPECS.items():
-        if pattern.search(lookup_key):
+        if pattern.search(last_component):
             return spec, pattern.pattern
     return None, None
 
@@ -143,9 +101,6 @@ if __name__ == "__main__":
         "Qwen/Qwen3-Omni-30B-A3B-Instruct",
         "Custom/Path/UnknownModel-Instruct",
         "Not/Matching/Anything",
-        "/path/to/MiniMax-H3/FL2VA",
-        "/path/to/MiniMax-H3/Ref2VA",
-        "MiniMaxAI/MiniMax-H3",
     ]
 
     test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
@@ -157,13 +112,7 @@ if __name__ == "__main__":
             if preprocessor := spec.get("payload_preprocessor"):
                 result = preprocessor(test_payload)
                 print(f"✓ {path:<40} → {result}")
-            elif params_builder := spec.get("params_builder"):
-                result = params_builder(
-                    {"audio_flow_shift": 3.0, "flow_shift": 12.0, "type": "minimax_h3"},
-                    extra_params={"task": "t2va"},
-                )
-                print(f"✓ {path:<40} → {result}")
             else:
-                print(f"✓ {path:<40} → No preprocessor/params_builder")
+                print(f"✓ {path:<40} → No preprocessor")
         else:
             print(f"✗ {path:<40} → No match")

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,8 +1,12 @@
 from collections.abc import Callable
 from enum import Enum, auto
-from typing import Any, Literal, TypeAlias
+from typing import (
-
+    Any,
-from typing_extensions import NotRequired, TypedDict
+    Literal,
+    NotRequired,
+    TypeAlias,
+    TypedDict,
+)
 
 AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"]
 
@@ -23,14 +27,6 @@ class WanModelSpecificParams(dict):
     pass
 
 
-class MiniMaxH3ModelSpecificParams(dict):
-    pass
-
-
-class VideoReferences(dict):
-    pass
-
-
 class ModelMode(Enum):
     IMAGE_GENERATION = auto()
     VIDEO_GENERATION = auto()
@@ -51,11 +47,9 @@ class ModelModeSpec(TypedDict):
 
 
 PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
-ParamsBuilder: TypeAlias = Callable[..., dict[str, Any]]
 
 
 class Spec(TypedDict):
     stages: list[Literal["diffusion", "autoregression"]]
     modes: list[ModelModeSpec]
     payload_preprocessor: NotRequired[PayloadPreprocessor]
-    params_builder: NotRequired[ParamsBuilder]

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json

@@ -1 +1,513 @@
-{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated
+{
+    "id": "3aea1d08-7de5-4997-ba71-b2f66ad72a21",
+    "revision": 0,
+    "last_node_id": 14,
+    "last_link_id": 28,
+    "nodes": [
+        {
+            "id": 10,
+            "type": "MarkdownNote",
+            "pos": [
+                227.99306819575278,
+                -231.24143306069843
+            ],
+            "size": [
+                240.3326478922474,
+                136.3505820791881
+            ],
+            "flags": {},
+            "order": 0,
+            "mode": 0,
+            "inputs": [],
+            "outputs": [],
+            "title": "Note: Task and Input",
+            "properties": {},
+            "widgets_values": [
+                "vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both text-to-video generation and image-to-video generation. The node will create corresponding payload depending on whether an input image is present or not."
+            ],
+            "color": "#432",
+            "bgcolor": "#000"
+        },
+        {
+            "id": 4,
+            "type": "LoadImage",
+            "pos": [
+                496.31859627609606,
+                -229.71277089860084
+            ],
+            "size": [
+                270,
+                314
+            ],
+            "flags": {},
+            "order": 1,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "image",
+                    "name": "image",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "image"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "choose file to upload",
+                    "name": "upload",
+                    "type": "IMAGEUPLOAD",
+                    "widget": {
+                        "name": "upload"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "IMAGE",
+                    "name": "IMAGE",
+                    "type": "IMAGE",
+                    "links": [
+                        25
+                    ]
+                },
+                {
+                    "localized_name": "MASK",
+                    "name": "MASK",
+                    "type": "MASK",
+                    "links": null
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "LoadImage"
+            },
+            "widgets_values": [
+                "cute-cat.jpg",
+                "image"
+            ]
+        },
+        {
+            "id": 8,
+            "type": "VLLMOmniDiffusionSampling",
+            "pos": [
+                465.5140218220927,
+                148.80072646828987
+            ],
+            "size": [
+                284.205078125,
+                226
+            ],
+            "flags": {},
+            "order": 2,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "n",
+                    "name": "n",
+                    "type": "INT",
+                    "widget": {
+                        "name": "n"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "num_inference_steps",
+                    "name": "num_inference_steps",
+                    "type": "INT",
+                    "widget": {
+                        "name": "num_inference_steps"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "guidance_scale",
+                    "name": "guidance_scale",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "guidance_scale"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "true_cfg_scale",
+                    "name": "true_cfg_scale",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "true_cfg_scale"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "vae_use_slicing",
+                    "name": "vae_use_slicing",
+                    "type": "BOOLEAN",
+                    "widget": {
+                        "name": "vae_use_slicing"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "vae_use_tiling",
+                    "name": "vae_use_tiling",
+                    "type": "BOOLEAN",
+                    "widget": {
+                        "name": "vae_use_tiling"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "seed",
+                    "name": "seed",
+                    "type": "INT",
+                    "widget": {
+                        "name": "seed"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "diffusion sampling params",
+                    "name": "diffusion sampling params",
+                    "type": "SAMPLING_PARAMS",
+                    "links": [
+                        27
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniDiffusionSampling"
+            },
+            "widgets_values": [
+                1,
+                50,
+                1,
+                1,
+                false,
+                false,
+                1992,
+                "randomize"
+            ]
+        },
+        {
+            "id": 14,
+            "type": "VLLMOmniWanParams",
+            "pos": [
+                518.6005521845747,
+                428.4455663719664
+            ],
+            "size": [
+                270,
+                106
+            ],
+            "flags": {},
+            "order": 3,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "guidance_scale_2",
+                    "name": "guidance_scale_2",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "guidance_scale_2"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "boundary_ratio",
+                    "name": "boundary_ratio",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "boundary_ratio"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "flow_shift",
+                    "name": "flow_shift",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "flow_shift"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "Wan video params",
+                    "name": "Wan video params",
+                    "type": "VIDEO_PARAMS",
+                    "links": [
+                        28
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniWanParams"
+            },
+            "widgets_values": [
+                4,
+                0.875,
+                5
+            ]
+        },
+        {
+            "id": 12,
+            "type": "VLLMOmniGenerateVideo",
+            "pos": [
+                827.2566336087867,
+                -73.77449831827545
+            ],
+            "size": [
+                400,
+                346
+            ],
+            "flags": {},
+            "order": 4,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "image",
+                    "name": "image",
+                    "shape": 7,
+                    "type": "IMAGE",
+                    "link": 25
+                },
+                {
+                    "localized_name": "sampling_params",
+                    "name": "sampling_params",
+                    "shape": 7,
+                    "type": "SAMPLING_PARAMS",
+                    "link": 27
+                },
+                {
+                    "localized_name": "lora",
+                    "name": "lora",
+                    "shape": 7,
+                    "type": "REMOTE_LORA",
+                    "link": null
+                },
+                {
+                    "localized_name": "model_params",
+                    "name": "model_params",
+                    "shape": 7,
+                    "type": "VIDEO_PARAMS",
+                    "link": 28
+                },
+                {
+                    "localized_name": "url",
+                    "name": "url",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "url"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "model",
+                    "name": "model",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "model"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "prompt",
+                    "name": "prompt",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "prompt"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "negative_prompt",
+                    "name": "negative_prompt",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "negative_prompt"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "width",
+                    "name": "width",
+                    "type": "INT",
+                    "widget": {
+                        "name": "width"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "height",
+                    "name": "height",
+                    "type": "INT",
+                    "widget": {
+                        "name": "height"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "fps",
+                    "name": "fps",
+                    "type": "INT",
+                    "widget": {
+                        "name": "fps"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "num_frames",
+                    "name": "num_frames",
+                    "type": "INT",
+                    "widget": {
+                        "name": "num_frames"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "video",
+                    "name": "video",
+                    "type": "VIDEO",
+                    "links": [
+                        26
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniGenerateVideo"
+            },
+            "widgets_values": [
+                "http://localhost:8000/v1",
+                "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+                "Make the cat in the attached video yawning. Eye closed and mouth wide open. Keep the original realistic style.",
+                "Cartoonish",
+                900,
+                1200,
+                12,
+                41
+            ]
+        },
+        {
+            "id": 13,
+            "type": "SaveVideo",
+            "pos": [
+                1257.5441369501657,
+                -44.12956394419863
+            ],
+            "size": [
+                374,
+                372.34896826553796
+            ],
+            "flags": {},
+            "order": 5,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "video",
+                    "name": "video",
+                    "type": "VIDEO",
+                    "link": 26
+                },
+                {
+                    "localized_name": "filename_prefix",
+                    "name": "filename_prefix",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "filename_prefix"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "format",
+                    "name": "format",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "format"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "codec",
+                    "name": "codec",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "codec"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [],
+            "properties": {},
+            "widgets_values": [
+                "video/ComfyUI",
+                "auto",
+                "auto"
+            ]
+        }
+    ],
+    "links": [
+        [
+            25,
+            4,
+            0,
+            12,
+            0,
+            "IMAGE"
+        ],
+        [
+            26,
+            12,
+            0,
+            13,
+            0,
+            "VIDEO"
+        ],
+        [
+            27,
+            8,
+            0,
+            12,
+            1,
+            "SAMPLING_PARAMS"
+        ],
+        [
+            28,
+            14,
+            0,
+            12,
+            3,
+            "VIDEO_PARAMS"
+        ]
+    ],
+    "groups": [
+        {
+            "id": 1,
+            "title": "Input",
+            "bounding": [
+                213.1706010087147,
+                -313.20095750667554,
+                560.8246471437027,
+                407.4246532472182
+            ],
+            "color": "#3f789e",
+            "font_size": 24,
+            "flags": {}
+        }
+    ],
+    "config": {},
+    "extra": {
+        "workflowRendererVersion": "LG",
+        "ds": {
+            "scale": 1.1469075819486894,
+            "offset": [
+                101.7632950847089,
+                492.29121889347806
+            ]
+        }
+    },
+    "version": 0.4
+}

benchmarks/diffusion/backends.py

@@ -1,6 +1,5 @@
 import asyncio
 import base64
-import json
 import mimetypes
 import os
 import time
@@ -14,17 +13,6 @@ from tqdm import tqdm
 DEFAULT_EDITS_BOT_TASK = "think"
 
 
-def _is_minimax_h3_mixed_reference(model: str) -> bool:
-    """MiniMax-H3 Ref2VA consumes reference videos as file paths.
-
-    Its reference preparer ffprobes / ffmpegs the reference file, so single
-    ``video_reference`` uploads are sent as a base64 data URL: the server
-    persists them to disk and keeps ``source_path``, handing the pipeline a
-    real file path instead of a decoded frame list.
-    """
-    return "MiniMax-H3" in (model or "")
-
-
 @dataclass
 class RequestFuncInput:
     prompt: str
@@ -367,21 +355,13 @@ async def async_request_v1_videos(
         )
     elif input.video_paths and len(input.video_paths) > 0:
         video_path = input.video_paths[0]
-        if _is_minimax_h3_mixed_reference(input.model):
+        image_file = open(video_path, "rb")
-            with open(video_path, "rb") as reference_file:
+        form.add_field(
-                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+            "input_reference",
-            form.add_field(
+            image_file,
-                "video_reference",
+            filename=os.path.basename(video_path),
-                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+            content_type=_guess_mime_type(video_path),
-            )
+        )
-        else:
-            image_file = open(video_path, "rb")
-            form.add_field(
-                "input_reference",
-                image_file,
-                filename=os.path.basename(video_path),
-                content_type=_guess_mime_type(video_path),
-            )
 
     job_id = None
     job_status = None

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -858,12 +858,7 @@ class RandomDataset(BaseDataset):
             height = int(self.args.height or profile.get("height") or 720)
             num_frames = int(self.args.num_frames or profile.get("num_frames") or 16)
             fps = float(self.args.fps or profile.get("fps") or 8)
-            # cv2 typically writes MPEG-4 Part 2 (``mp4v``), which pipelines such
+            writer = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
-            # as MiniMax-H3 Ref2VA reject because they require H.264/H.265
-            # reference videos. Write the synthetic clip first, then re-encode
-            # it to H.264 when ffmpeg is available; otherwise keep ``mp4v``.
-            raw_path = video_path + ".raw.mp4"
-            writer = cv2.VideoWriter(raw_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
             if not writer.isOpened():
                 raise RuntimeError("cv2.VideoWriter failed to open")
             for frame_idx in range(num_frames):
@@ -876,10 +871,6 @@ class RandomDataset(BaseDataset):
                 frame[max(y - 36, 0) : min(y + 36, height), x : min(x + 96, width), :] = (64, 128, 220)
                 writer.write(frame)
             writer.release()
-            if _reencode_video_to_h264(raw_path, video_path):
-                os.unlink(raw_path)
-            else:
-                os.replace(raw_path, video_path)
             return [video_path]
         except Exception as e:
             raise RuntimeError(
@@ -887,40 +878,6 @@ class RandomDataset(BaseDataset):
             ) from e
 
 
-def _reencode_video_to_h264(source: str, destination: str) -> bool:
-    """Re-encode a video to H.264/MP4 using ffmpeg when available.
-
-    Returns False when ffmpeg is missing or the re-encode fails so callers can
-    fall back to the original container/codec.
-    """
-    import shutil
-    import subprocess
-
-    if shutil.which("ffmpeg") is None:
-        return False
-    try:
-        result = subprocess.run(
-            [
-                "ffmpeg",
-                "-y",
-                "-loglevel",
-                "error",
-                "-i",
-                source,
-                "-c:v",
-                "libx264",
-                "-pix_fmt",
-                "yuv420p",
-                destination,
-            ],
-            capture_output=True,
-            text=True,
-        )
-        return result.returncode == 0 and os.path.exists(destination)
-    except Exception:
-        return False
-
-
 def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None:
     """Compute expected execution time (ms) based on a base per-step-per-frame unit time.
 

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.25.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm
@@ -26,10 +26,6 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
 
-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.25.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm
@@ -26,10 +26,6 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
     python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation
 
-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 ENTRYPOINT []

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.25.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,7 +35,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "psutil>=7.2.0" \
       "soundfile>=0.13.1" \
       "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
       "opencv-python>=4.12.0.88" \
       "imageio[ffmpeg]>=0.6.0" \
       "jiwer>=3.0.0" \

docker/Dockerfile.npu.ci.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.25.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,7 +35,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "psutil>=7.2.0" \
       "soundfile>=0.13.1" \
       "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
       "opencv-python>=4.12.0.88" \
       "imageio[ffmpeg]>=0.6.0" \
       "jiwer>=3.0.0" \
@@ -45,7 +44,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "hyperpyyaml>=1.2.0" \
       "onnx>=1.16.0" \
       "librosa>=0.10.0" \
-      "torchmetrics>=1.9.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

docs/.nav.yml

@@ -68,12 +68,13 @@ nav:
         - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
         - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
         - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload.md
+      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
       - LoRA: user_guide/diffusion/lora.md
       - Custom Pipeline: features/custom_pipeline.md
-      - Execution Modes: user_guide/diffusion/execution_modes.md
+      - Request-Level Batching: user_guide/diffusion/request_batching.md
+      - Step Execution: user_guide/diffusion/step_execution.md
     - Quantization:
-      - Quantization: user_guide/quantization/overview.md
+      - Overview: user_guide/quantization/overview.md
       - Online Quantization: user_guide/quantization/online.md
       - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
       - FP8 W8A8: user_guide/quantization/fp8.md
@@ -101,65 +102,31 @@ nav:
   - CI: contributing/ci
   - Design Documents:
     - design/index.md
-    - Architecture Documents:
+    - design/architecture_overview.md
-      - Architecture Overview: design/architecture_overview.md
     - Feature Design:
-      - Runtime and Stage Execution:
+      - design/feature/disaggregated_inference.md
-        - Disaggregated Inference: design/feature/disaggregated_inference.md
+      - design/feature/ray_based_execution.md
-        - Async Chunk: design/feature/async_chunk.md
+      - design/feature/omni_connectors/
-        - Async Diffusion Output: design/feature/async_diffusion_output.md
+      - design/feature/prefix_caching.md
-        - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
+      - design/feature/cfg_parallel.md
-        - Automatic Prefix Caching: design/feature/prefix_caching.md
+      - design/feature/expert_parallel.md
-      - Communication:
+      - design/feature/sequence_parallel.md
-        - OmniConnector Implementations:
+      - design/feature/tensor_parallel.md
-          - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
+      - design/feature/vae_parallel.md
-          - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
+      - design/feature/hsdp.md
-          - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
+      - design/feature/cache_dit.md
-          - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
+      - design/feature/teacache.md
-          - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
+      - design/feature/skip_softmax.md
-          - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
+      - design/feature/async_chunk.md
-      - Diffusion Acceleration:
+      - design/feature/vae_parallel.md
-        - Parallelism:
+      - design/feature/diffusion_step_execution.md
-          - CFG-Parallel: design/feature/cfg_parallel.md
+      - design/feature/diffusion_request_level_batching.md
-          - Expert Parallel: design/feature/expert_parallel.md
+      - design/feature/diffusion_continuous_batching.md
-          - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
-          - Pipeline Parallel: design/feature/pipeline_parallel.md
-          - Sequence Parallel: design/feature/sequence_parallel.md
-          - Tensor Parallel: design/feature/tensor_parallel.md
-          - VAE Patch Parallelism: design/feature/vae_parallel.md
-        - Attention Backends:
-          - Skip-Softmax: design/feature/skip_softmax.md
-        - Quantization: design/feature/quantization.md
-        - Cache-DiT: design/feature/cache_dit.md
-        - TeaCache: design/feature/teacache.md
-        - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
-        - Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md
-    - Infrastructure and Performance:
-      - Prometheus Metrics: design/metrics.md
-      - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
     - Module Design:
-      - Entrypoints: design/module/entrypoints.md
+      - design/module/ar_module.md
-      - vLLM-Omni Configuration: design/module/vllm_omni_config.md
+      - design/module/dit_module.md
-      - Input, Output, and Modality Contracts: design/module/input_output_modality_contracts.md
+      - design/module/entrypoint_module.md
-      - Error Contracts: design/module/error_contracts.md
+      - design/module/async_omni_architecture.md
-      - Engine Orchestration: design/module/engine_orchestration.md
-      - Stage Runtime: design/module/stage_runtime.md
-      - OmniConnector: design/module/omni_connector.md
-      - Model Integration: design/module/model_integration.md
-      - Autoregressive Runtime: design/module/ar_runtime.md
-      - Diffusion:
-        - Overview: design/module/diffusion/index.md
-        - Runtime: design/module/diffusion/diffusion_runtime.md
-        - Model Integration: design/module/diffusion/diffusion_model_integration.md
-        - Continuous Batching: design/module/diffusion/continuous_batching.md
-        - Parallelism: design/module/diffusion/parallelism.md
-        - Offloader: design/module/diffusion/offloader.md
-      - Execution Platforms: design/module/execution_platforms.md
-      - Cache Management: design/module/cache_management.md
-      - Quantization: design/module/quantization.md
-      - Observability: design/module/observability.md
-      - Profiling: design/module/profiling.md
-      - Benchmarking: design/module/benchmarking.md
   - Docs Guide: contributing/DOCS_GUIDE.md
 - API Reference:
   - api/README.md

docs/README.md

@@ -52,13 +52,12 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
-- Full-duplex realtime serving with streaming audio input and output (experimental)
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
+- Omni-modality models (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
 - TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
+- Diffusion models — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
 - Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
 
 For more information, checkout the following:

docs/cli/README.md

@@ -18,12 +18,18 @@ Specify the port:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-Load a custom deploy configuration with `--deploy-config`:
+For a migrated model, load a custom deploy configuration with `--deploy-config`:
 
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --deploy-config /path/to/deploy_config.yaml
 ```
 
+The deprecated `--stage-configs-path` flag is retained for models that still use the legacy `stage_args` schema:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --stage-configs-path /path/to/legacy_stage_config.yaml
+```
+
 ## bench
 
 Run benchmark tests for online serving throughput.

docs/cli/serve.md

@@ -8,6 +8,7 @@ The stage-based CLI is designed for deployments that require launching each pipe
 - For **migrated models** that utilize the bundled deployment YAML configurations located in
   `vllm_omni/deploy/`, the `--deploy-config` flag is only required to override the default configuration. By default, executing `vllm serve MODEL --omni ...`
   automatically loads the bundled deployment configuration.
+- For custom legacy `stage_args` YAMLs, pass the file with `--stage-configs-path`.
 
 Example: Initializing Stage 0 (Orchestrator and API Server):
 The commands below show a common device mapping where Stage 0 uses GPU 0 and
@@ -31,7 +32,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-When utilizing a custom deployment YAML, append `--deploy-config /path/to/override.yaml` to each command execution.
+When utilizing a custom deployment YAML based on the new schema, append `--deploy-config /path/to/override.yaml` to each command execution. Conversely, for legacy models, substitute this parameter with `--stage-configs-path /path/to/stage_configs.yaml`.
 
 In the standard execution paradigm, the `--stage-overrides` argument is utilized to apply stage-specific configurations from a single CLI command.
 However, under the **stage-based CLI** paradigm, where each process strictly encapsulates a single stage, it is recommended to specify tuning parameters directly via discrete command-line flags for the respective stage, rather than constructing a composite `--stage-overrides` JSON string.

docs/community/governance.md

@@ -40,7 +40,6 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@Isotr0py](https://github.com/Isotr0py): Diffusion and Quantization
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
-- [@NickCao](https://github.com/NickCao): CI, Tests, Entrypoint, and TTS Model Support
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
 - [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
 - [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
+For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.16.0/configuration/index.html)
 
 Currently, the main options are maintained by stage configs for each model.
 

docs/configuration/composable_parallel.md

@@ -34,8 +34,12 @@ Two additional `StrategySpec` slots — `layer_hook_specs` and `kernel_specs`
 | Flag | Description |
 |------|-------------|
 | `--strategy-config PATH` | Path to a `strategy.yaml`. Only takes effect on registry-based deploy paths (`--deploy-config` or the bundled default deploy YAML). Its derived sizing is overlaid onto the merged stages *before* CLI overrides. |
+| `--stage-configs-path PATH` | **Legacy.** Loads the old `stage_args` YAML schema for un-migrated models. Passing `--strategy-config` together with this path is silently inapplicable — vLLM-Omni emits a warning and the strategy is ignored (see `vllm_omni/entrypoints/utils.py`, lines 372–380). |
 | `--omni-lb-policy POLICY` | Orchestrator-wide load-balancer policy. Strategy-derived from any `stage_replica` axis; an explicit CLI value wins and must match the derived one or `AsyncOmniEngine` raises `Conflicting load-balancer policy` at construction. |
 
+!!! warning
+    `--strategy-config` cannot be used with `--stage-configs-path` (the legacy deploy path). If your model has not been migrated to the registry-based pipeline yet, the strategy is silently dropped with a warning. Migrate to a registry-based model or use `--deploy-config` to apply a strategy.
+
 ## Sub-schemas
 
 ### `axis`
@@ -212,6 +216,17 @@ Strategy keys MUST be `model_stage` *names* (strings), the same labels stage con
 !!! warning "Mixing CLI overrides with a strategy on the same axis"
     A strategy is meant to be the single writer for the axes it declares. If a CLI override (`--tensor-parallel-size 2`, `--stage-overrides '{"0": {"tensor_parallel_size": 2}}'`) sets the same axis, the **CLI value wins** and a warning is emitted from `_reconcile_strategy_with_cli`. The device-layout guard then re-runs against the effective layout, so a CLI override that breaks `tp * dp * pp * num_replicas` will still fail fast at config time.
 
+!!! warning "Passing `--strategy-config` against the legacy `--stage-configs-path`"
+    Composable parallel only applies on the registry-based deploy path. If a model still resolves through the legacy `stage_args` YAML (loaded via `--stage-configs-path`), `--strategy-config` is silently inapplicable and emits:
+
+    ```text
+    --strategy-config (<path>) was provided but model '<model>' resolves via the
+    legacy stage_configs YAML path, which does not support composable-parallel
+    strategies; the strategy is ignored. Use a registry-based model to apply it.
+    ```
+
+    Migrate the model to the new deploy schema (see `configuration/stage_configs.md`) before applying a strategy.
+
 !!! warning "Conflicting `omni_lb_policy` across stages"
     `omni_lb_policy` is a single pipeline-wide knob. If two `stage_replica` axes in different stages derive different policies, `apply_strategy_specs` raises:
 

docs/configuration/pd_disaggregation.md

@@ -158,7 +158,7 @@ runtime:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-    --deploy-config /path/to/qwen3_omni_pd.yaml
+    --stage-configs-path /path/to/qwen3_omni_pd.yaml
 ```
 
 ## Operational Notes

docs/configuration/stage_configs.md

@@ -1,9 +1,9 @@
-# Pipeline and deploy configurations
+# Deploy and legacy stage configurations
 
-In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run.
+In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run. Models that have not migrated to this split still use legacy stage configurations with a `stage_args` schema.
 
 !!! note
-    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when `--deploy-config` is omitted. The model registry resolves the right pipeline and deploy YAML by `model_type`.
+    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when neither `--stage-configs-path` nor `--deploy-config` is provided — the model registry resolves the right pipeline + deploy YAML by `model_type`. Custom legacy `stage_args` YAMLs are still accepted via `--stage-configs-path`.
 
 ## Deploy configuration schema
 
@@ -86,9 +86,10 @@ stages:
 
 | Flag | Description |
 |------|-------------|
-| `--deploy-config PATH` | Load a deploy YAML. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
+| `--deploy-config PATH` | Load a new-schema deploy YAML. It is mutually exclusive with `--stage-configs-path`. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
 | `--stage-overrides JSON` | Per-stage JSON overrides, e.g. `'{"0":{"gpu_memory_utilization":0.5}}'`. Per-stage values always win over global flags. |
 | `--async-chunk` / `--no-async-chunk` | Flip the deploy YAML's `async_chunk:` bool. Unset (default) leaves the YAML value in force. |
+| `--stage-configs-path` | **Deprecated.** Load a legacy `stage_args` YAML. New-schema YAMLs passed to this flag are auto-detected for compatibility, but should use `--deploy-config`. It is mutually exclusive with `--deploy-config` and will be removed in a future release. |
 
 ### Stage-Based CLI Paradigm
 
@@ -118,7 +119,8 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --omni-master-port 26000
 ```
 
-When instantiating a custom deployment YAML, append the `--deploy-config /path/to/override.yaml` directive to all node invocations.
+When instantiating a custom deployment YAML conforming to the updated schema, append the `--deploy-config /path/to/override.yaml` directive
+to all node invocations. For legacy architectures (e.g., BAGEL) configured via deprecated `stage_args:` schemas, continue to specify the relevant configuration via `--stage-configs-path /path/to/config.yaml`.
 
 In the context of standard initialization architectures, utilizing the `--stage-overrides` parameter operates as the optimal methodology
 for delineating stage-specific tuning from the CLI interface:
@@ -218,7 +220,8 @@ Therefore, as a core part of vLLM-Omni, a model's pipeline and deployment config
 
 To override specific parameters, explicitly inject the customized configuration schema
 in both online and offline instantiation flows. Use the `--deploy-config` flag
-when loading a deploy configuration.
+when loading a deploy configuration, reserving the `--stage-configs-path` parameter
+exclusively to maintain compatibility with legacy `stage_args` YAML constructs.
 
 Examples:
 
@@ -233,6 +236,11 @@ For online serving:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```
 
+Legacy online serving:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
 !!! important
     We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.
 
@@ -387,8 +395,8 @@ that support request-level batching with `step_execution` disabled.
 
 Use this together with `max_num_seqs > 1` for bursty serving traffic. `0`
 disables admission waiting and preserves the lowest first-request latency.
-For diffusion execution and batching tuning, see
+For diffusion request-level batching tuning, see
-[Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
+[Request-Level Batching](../user_guide/diffusion/request_batching.md).
 
 Default: `0.0`
 

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -1,6 +1,6 @@
 **Scope**
 
-For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
+For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
 
 **Test Case Design**
 

docs/contributing/ci/test_examples/l4_performance_tests.inc.md

@@ -44,11 +44,9 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
             "ignore_eos": true,
             "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
             "baseline": {
-                "H100": {
+                "mean_ttft_ms": [500, 800],
-                    "mean_ttft_ms": [500, 800],
+                "mean_audio_ttfp_ms": [2000, 3500],
-                    "mean_audio_ttfp_ms": [2000, 3500],
+                "mean_audio_rtf": [0.25, 0.35]
-                    "mean_audio_rtf": [0.25, 0.35]
-                }
             }
         }
     ]
@@ -144,6 +142,10 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
   --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
 pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
   --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+
+# Optional baseline assertion (default off)
+pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
+  --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 ```
 
 See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
@@ -200,4 +202,4 @@ You can add any benchmark running parameters you need here. For all optional par
 | num_prompts     | int / array | Yes      | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
 | request_rate    | float / array | No  | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
 | max_concurrency | int / array | No       | 1, [1, 2, 3]    | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode).                                                                                                                                                                                                             |
-| baseline        | object      | No       | see below       | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. |
+| baseline        | object      | No       | see above       | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).  

docs/contributing/ci/test_system_overview.md

@@ -56,11 +56,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td>/</td>
       <td><code>core_model and cpu</code></td>
       <td rowspan="2">&lt;15min</td>
-      <td>
+      <td>/tests/{component_name}/test_xxx</td>
-        <code>tests/{component}/…</code> mirroring <code>vllm_omni/{component}/</code><br>
-        (e.g. <code>tests/diffusion/</code>, <code>tests/engine/</code>, <code>tests/entrypoints/</code>)<br>
-        Do <strong>not</strong> add new top-level <code>tests/</code> dirs unrelated to a component.
-      </td>
       <td>
         <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
         Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
@@ -72,18 +68,16 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
       <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
       <td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
-      <td>High-priority features and component tests that require launching instances</td>
+      <td>High-priority features (using random lightweight models)</td>
       <td>High-priority interfaces (using random lightweight models)</td>
       <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>
-        <strong>Model E2E:</strong><br>
+        <strong>Model tests:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}.py</code><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
-        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
+        <strong>Feature tests:</strong><br>
-        <strong>Feature integration:</strong><br>
+        /tests/{component_name}/test_xxx<br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        <strong>Interface tests:</strong><br>
-        (e.g. <code>fullduplex/</code>, <code>custom_pipeline/</code>, <code>rlhf_test/</code>)<br>
+        /tests/entrypoints/test_xxx
-        <strong>Component / interface Test (GPU):</strong><br>
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
@@ -96,19 +90,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
       <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
       <td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
-      <td>Medium-priority features and component tests that require launching instances</td>
+      <td>Medium-priority features (using random lightweight models)</td>
       <td>Medium-priority interfaces (using random lightweight models)</td>
       <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;30min</td>
       <td>
-        <strong>Model E2E:</strong><br>
+        <strong>Model tests:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}.py</code><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
-        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
+        /tests/e2e/offline_inference/test_{model_name}.py<br>
-        <code>tests/e2e/accuracy/</code><br>
+        <strong>Feature tests:</strong><br>
-        <strong>Feature integration:</strong><br>
+        /tests/{component_name}/test_xxx<br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        <strong>Interface tests:</strong><br>
-        <strong>Component / interface Test:</strong><br>
+        /tests/entrypoints/test_xxx
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
       </td>
       <td>
         <a href="test_writing_guide.md#l3-level-testing-core-integration-performance-and-accuracy-verification">L3</a><br>
@@ -121,23 +114,26 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
       <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
       <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
-      <td>Low-priority features and component tests that require launching instances</td>
+      <td>Low-priority features (using real weights)</td>
       <td>Low-priority interfaces (using real weights)</td>
       <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
       <td>&lt;3 hour</td>
       <td>
-        <strong>Model E2E:</strong><br>
+        <strong>Model tests:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}_expansion.py</code><br>
+        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        <code>tests/e2e/offline_inference/test_{model}_expansion.py</code><br>
+        <strong>Feature tests:</strong><br>
-        <code>tests/e2e/accuracy/test_{model}.py</code><br>
+        /tests/{component_name}/test_xxx<br>
-        <strong>Feature integration:</strong><br>
+        <strong>Interface tests:</strong><br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
+        /tests/entrypoints/test_xxx<br>
-        <strong>Component / interface Test:</strong><br>
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code><br>
         <strong>Performance:</strong><br>
-        <code>tests/dfx/perf/tests/</code><br>
+        /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),<br>
-        <strong>Doc examples:</strong><br>
+        test_voxcpm2.json, test_higgs_audio_v3.json, and<br>
-        <code>tests/examples/online_serving/</code>, <code>tests/examples/offline_inference/</code>
+        /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
+        <strong>Doc Test:</strong><br>
+        tests/examples/online_serving/test_{model_name}.py<br>
+        tests/examples/offline_inference/test_{model_name}.py<br>
+        <strong>Accuracy Test:</strong><br>
+        /tests/e2e/accuracy/test_{model_name}.py
       </td>
       <td>
         <a href="test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing">L4</a><br>
@@ -156,9 +152,11 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
       <td> Depends on reality </td>
       <td>
         <strong>Stability:</strong><br>
-        <code>tests/dfx/stability/tests/</code><br>
+        /tests/dfx/stability/tests/test_qwen3_omni.json<br>
+        /tests/dfx/stability/tests/test_wan22.json<br>
         <strong>Reliability:</strong><br>
-        <code>tests/dfx/reliability/test_reliability_{model}.py</code>
+        tests/dfx/reliability/test_reliability_{model_key}.py<br>
+        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
       </td>
       <td>
         <a href="test_writing_guide.md#l5-level-testing-stability-and-reliability-testing">L5</a><br>
@@ -170,9 +168,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
   </tbody>
 </table>
 
-**Test Dir placement (summary):** component / unit under `tests/{component}/` mirroring `vllm_omni/`; model E2E under `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, and `tests/e2e/accuracy/`; feature integration under `tests/e2e/features/<feature>/`; doc example tests under `tests/examples/online_serving/` and `tests/examples/offline_inference/`; performance under `tests/dfx/perf/`; stability under `tests/dfx/stability/`; reliability under `tests/dfx/reliability/`. Do **not** add new top-level directories under `tests/` that are unrelated to a `vllm_omni` component (or to the established `e2e` / `dfx` / `helpers` / `examples` / `buildkite` layout).
+For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md).
-
-For per-level test authoring (markers, examples), see [Test Writing Guide](./test_writing_guide.md).
 
 ## Common Specifications
 

docs/contributing/ci/test_writing_guide.md

@@ -4,20 +4,137 @@ This document describes how to author tests for each CI level (L1–L5). For the
 
 ## Test Directory Structure
 
-Canonical placement by CI level is in the **Test Dir** column of the level matrix in [Test System Overview](./test_system_overview.md).
+Legend: `✅` = test exists, `⬜` = suggested to add.
+```
+vllm_omni/                                    tests/
+├── config/                             →     ├── config/
+│   ├── model.py                              │   └── test_model.py                    ⬜
+│   └── lora.py                               │   └── test_lora.py                      ⬜
+│
+├── core/                               →     ├── core/
+│   └── sched/                                 │   └── sched/
+│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
+│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
+│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
+│
+├── diffusion/                          →     ├── diffusion/
+│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
+│   ├── attention/                             │   ├── attention/
+│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
+│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
+│   ├── distributed/                           │   ├── distributed/
+│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
+│   │                                            │   │   ├── test_cfg_parallel.py        ✅
+│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
+│   ├── lora/                                   │   ├── lora/
+│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
+│   │                                            │   │   └── test_lora_manager.py        ✅
+│   ├── models/                                 │   ├── models/
+│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
+│   │   ├── z_image/                            │   │   └── z_image/
+│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
+│   └── worker/                                 │   └── worker/
+│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
+│       └── diffusion_model_runner.py            │
+│
+├── distributed/                         →     ├── distributed/
+│   └── omni_connectors/                         │   └── omni_connectors/
+│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
+│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
+│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
+│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
+│
+├── engine/                             →     ├── engine/
+│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
+│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
+│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
+│
+├── entrypoints/                        →     ├── entrypoints/
+│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
+│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
+│   │   └── ...                                  │   │   └── test_*.py                  ⬜
+│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
+│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
+│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
+│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
+│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
+│
+├── inputs/                             →     ├── inputs/
+│   ├── data.py                                 │   ├── test_data.py                   ⬜
+│   ├── parse.py                                │   ├── test_parse.py                 ⬜
+│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
+│
+├── model_executor/                     →     ├── model_executor/
+│   ├── layers/                                  │   ├── layers/
+│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
+│   ├── model_loader/                            │   ├── model_loader/
+│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
+│   ├── models/                                  │   ├── models/
+│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
+│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
+│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
+│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
+│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
+│   │       └── ...                               │   │   └── qwen3_omni/
+│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
+│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
+│   └── stage_input_processors/                  │   └── stage_input_processors/
+│       └── ...                                  │       └── test_*.py                 ⬜
+│
+├── sample/                             →     ├── sample/
+│   └── __init__.py                             │   └── test_*.py                      ⬜
+│
+├── utils/                              →     ├── utils/
+│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
+│
+├── worker/                             →     ├── worker/
+│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
+│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
+│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
+│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
+│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
+│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
+│
+├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
+│   ├── cuda/
+│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
+│   ├── rocm/
+│   └── xpu/worker/
+│
+├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
+├── (logger, patch, request, version)    →     (no corresponding unit test)
+│
+└── e2e (tests side only)               →     ├── e2e/
+                                               ├── online_serving/                     ✅ non-empty
+                                               │   ├── test_qwen2_5_omni.py
+                                               │   ├── test_async_omni.py
+                                               │   ├── test_qwen3_omni.py
+                                               │   ├── test_qwen3_omni_expansion.py
+                                               │   ├── test_mimo_audio.py
+                                               │   └── test_images_generations_lora.py
+                                               └── offline_inference/                  ✅
+                                                   ├── test_qwen2_5_omni.py
+                                                   ├── test_qwen3_omni.py
+                                                   ├── test_bagel_text2img.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
+                                                   ├── test_zimage_tensor_parallel.py
+                                                   ├── test_cache_dit.py
+                                                   ├── test_teacache.py
+                                                   ├── test_stable_audio_expansion.py
+                                                   ├── test_diffusion_cpu_offload.py
+                                                   ├── test_diffusion_layerwise_offload.py
+                                                   ├── test_diffusion_lora.py
+                                                   ├── test_sequence_parallel.py
+                                                   └── stage_configs/                  (legacy schema, still
+                                                       ├── bagel_*.yaml                 present for unmigrated
+                                                       └── npu/, rocm/, etc.            models)
 
-Summary:
+# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under
-
+# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md.
-- **Component / unit:** `tests/{component}/…` mirroring `vllm_omni/{component}/`
+```
-- **Model E2E:** `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, `tests/e2e/accuracy/`
-- **Feature integration:** `tests/e2e/features/<feature>/`
-- **Doc example tests:** `tests/examples/online_serving/`, `tests/examples/offline_inference/`
-- **Performance:** `tests/dfx/perf/tests/` (JSON configs; runners under `tests/dfx/perf/scripts/`)
-- **Stability:** `tests/dfx/stability/tests/`
-- **Reliability:** `tests/dfx/reliability/`
-- Do **not** add new top-level `tests/` directories unrelated to a component (or to `e2e` / `dfx` / `helpers` / `examples` / `buildkite`)
-
-Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [stage configs](../../configuration/stage_configs.md)).
 
 ## Markers for Tests
 
@@ -256,13 +373,13 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that
 
 #### 1.3 Test Directory and Execution Files
 
-A clear directory structure is key to managing test cases efficiently. See the **Test Dir** column in [Test System Overview](./test_system_overview.md) for the canonical placement rules.
+A clear directory structure is key to managing test cases efficiently.
 
--   ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component.
+-   ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py`
--   ***L2 (basic model E2E)***:
+-   -   Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file.
-    -   Online serving: `tests/e2e/online_serving/test_{model_name}.py`
+-   ***L2 Test Directory***:
-    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}.py`
+-   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
--   ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features/<feature>/` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`).
+    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`
 
 #### 1.4 Execution Method and Example
 
@@ -368,12 +485,10 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
 #### 2.3 Test Directory and Execution Files
 
--   ***Functional / model E2E***:
+-   ***Functional Testing***:
-    -   Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py`
+-   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
-    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py`
+    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
-    -   Accuracy / similarity: `tests/e2e/accuracy/`
+    -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).
-    -   (`_expansion.py` holds broader scenario coverage than the L2 smoke file.)
--   ***Feature-level integration***: `tests/e2e/features/<feature>/` (not under a new top-level `tests/` directory).
 
 #### 2.4 Execution Method and Example
 

docs/contributing/model/adding_diffusion_model.md

@@ -653,7 +653,7 @@ python end2end.py --model your-org/your-model-name --modality text2img --prompts
 Mirror BAGEL’s online serving setup:
 
 - Server launcher: `examples/online_serving/your_model_name/run_server.sh`
-  - Wrap `vllm serve ... --omni --port ...` (and `--deploy-config ...` if needed)
+  - Wrap `vllm serve ... --omni --port ...` (and `--stage-configs-path ...` if needed)
 - Client: `examples/online_serving/your_model_name/openai_chat_client.py`
   - Send requests to `POST /v1/chat/completions`
   - Support multimodal inputs (e.g., base64 image) if your model needs it
@@ -755,8 +755,7 @@ omni = Omni(model="your-model", ulysses_degree=2, ring_degree=2)
 
 ### Step Execution
 
-See the detailed
+See detailed design guide: [How to add step execution support](../../design/feature/diffusion_step_execution.md)
-[Diffusion Continuous Batching design guide](../../design/feature/diffusion_continuous_batching.md).
 
 Use this only when your pipeline can be split into stable request-scoped and
 step-scoped phases. The reference implementation is
@@ -770,8 +769,9 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
-The same design guide covers the experimental batched step-wise path used when
+If you want the pipeline to work with the experimental batched step-wise path
-`max_num_seqs > 1`.
+(`max_num_seqs > 1`), also see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
 
 If you expose this in example scripts or recipes, keep it opt-in. Surface
 runtime features like `step_execution` as optional flags instead of silently
@@ -824,7 +824,7 @@ omni = Omni(model="your-model",
 
 ### CPU Offload
 
-See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload.md)
+See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload_diffusion.md)
 
 vLLM-Omni provides two offloading strategies to reduce GPU memory usage:
 

docs/contributing/model/adding_omni_model.md

@@ -48,8 +48,8 @@ vllm_omni/deploy/
 ```
 
 New in-tree models should register their `PipelineConfig` in
-`vllm_omni/config/pipeline_registry.py`; use `--deploy-config` for custom
+`vllm_omni/config/pipeline_registry.py`; use `--stage-configs-path` only for
-deployment YAMLs.
+custom legacy `stage_args` YAMLs.
 
 ## Step-by-Step Implementation
 

docs/contributing/model/adding_tts_model.md

@@ -449,8 +449,9 @@ Key configuration fields:
 !!! note
     New in-tree models should define frozen topology in `models/<model>/pipeline.py`,
     register it in `vllm_omni/config/pipeline_registry.py`, and put deployment
-    knobs in `vllm_omni/deploy/<model>.yaml`. Use `--deploy-config` to load a
+    knobs in `vllm_omni/deploy/<model>.yaml`. Legacy `stage_args` YAMLs are still
-    custom deployment YAML.
+    accepted for custom configs via `--stage-configs-path`, but should not be used
+    for new bundled defaults.
 
 ### Batch mode
 

docs/design/feature/async_diffusion_output.md

@@ -1,174 +0,0 @@
-# Async Diffusion Output
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Performance](#performance)
-3. [Architecture](#architecture)
-4. [Model Coverage](#model-coverage)
-5. [Configuration](#configuration)
-6. [Related Files](#related-files)
-
-## Overview
-
-The async diffusion output feature moves the D2H (Device→Host) copy and SHM packing from the Worker main thread to a background daemon thread, so the GPU default stream can immediately start the next request's forward pass. This eliminates the GPU bubble caused by synchronous D2H/packing.
-
-**Core benefit**: the previous request's output D2H/packing overlaps with the next request's forward pass.
-
-**Automatically enabled** when `step_execution=False` (default, request-mode). No extra configuration needed. When `step_execution=True` (step-mode), the original synchronous path is used — pump thread and Worker background thread are not started.
-
-## Performance
-
-### HunyuanImage-3.0 (TP4)
-
-| Resolution | Async Off (QPS) | Async On (QPS) | Change |
-|---|---|---|---|
-| 1024×1024 | 0.4773 | 0.4802 | +0.60% |
-| 768×768 | 0.8370 | 0.8533 | +1.95% |
-
-**Analysis**: smaller resolution → larger QPS gain. D2H/packing is relatively fixed overhead; forward is variable (scales with resolution). Smaller images → shorter forward → D2H/packing occupies a larger fraction of total time → more benefit from overlapping it.
-
-## Architecture
-
-### Execution Timeline
-
-```
-Before (synchronous D2H):
-[forward req1] [D2H+SHM pack req1] [forward req2] [D2H+SHM pack req2]
-                                 ^^^^^^^^^^^^^^^^ GPU bubble
-
-After (async D2H):
-[forward req1] [forward req2]
-[D2H+SHM pack req1 (side stream)] [D2H+SHM pack req2 (side stream)]
-                                 ^^^^^^^^^^^^^^^^ bubble eliminated
-```
-
-### Data Flow
-
-```
-                    ┌─────────────────────────────────────────────────────┐
-                    │                 Worker Process                      │
-                    │                                                    │
-  execute_model ──> │ WorkerBusyLoop ──> forward() ──> DiffusionOutput   │
-                    │       │                              │ (GPU tensor)│
-                    │       │                    ┌─────────┴──────────┐  │
-                    │       │                    │  async output path │  │
-                    │       │                    │                    │  │
-                    │       │              compute_done ──> result_mq │  │
-                    │       │                    │                    │  │
-                    │       │                    ▼                    │  │
-                    │       │            AsyncOutputThread            │  │
-                    │       │         (side CUDA stream)             │  │
-                    │       │         wait_event(gpu_event)          │  │
-                    │       │         pack_diffusion_output_shm()    │  │
-                    │       │         D2H + SHM write                │  │
-                    │       │                    │                    │  │
-                    │       │              output_ready ──> result_mq │  │
-                    │       │                    │                    │  │
-                    │       ▼                    ▼                    │  │
-                    │  (dequeue next request)                        │  │
-                    └─────────────────────────────────────────────────────┘
-                                         │
-                                         ▼
-                    ┌─────────────────────────────────────────────────────┐
-                    │              Executor (main process)               │
-                    │                                                    │
-                    │  ResultPumpThread (sole reader of result_mq)       │
-                    │       │                                            │
-                    │       ├── AsyncDiffusionOutput(COMPUTE_DONE)       │
-                    │       │   → resolve _rpc_futures[rpc_id]           │
-                    │       │   → RunnerOutput(async_output_id=...)      │
-                    │       │                                            │
-                    │       ├── AsyncDiffusionOutput(OUTPUT_READY)       │
-                    │       │   → resolve _output_futures[async_output_id]│
-                    │       │   → or batch split via _batch_split_map   │
-                    │       │                                            │
-                    │       └── Non-async message                        │
-                    │           → _sync_result_buffer (for other RPCs)   │
-                    │                                                    │
-                    │  wait_output_ready(async_output_id) → Future       │
-                    └─────────────────────────────────────────────────────┘
-```
-
-### Key Components
-
-1. **AsyncDiffusionOutput** (`data.py`): Protocol envelope for `result_mq`. `kind` field routes messages:
-   - `COMPUTE_DONE` — forward finished, GPU can start next request
-   - `OUTPUT_READY` — D2H/SHM packing finished, final output available
-   - `RPC_RESULT` — ordinary RPC return (including error propagation)
-
-2. **Worker AsyncOutputThread** (`diffusion_worker.py`): Background daemon thread that:
-   - Waits for `gpu_event` (cross-stream sync ensuring forward wrote the tensor)
-   - Runs `pack_diffusion_output_shm()` on a side CUDA stream
-   - Enqueues `OUTPUT_READY` to `result_mq` when packing completes
-
-3. **ResultPumpThread** (`multiproc_executor.py`): Sole reader of `result_mq` in request-mode. Dispatches messages:
-   - `COMPUTE_DONE` / `RPC_RESULT` → resolves `_rpc_futures[rpc_id]`
-   - `OUTPUT_READY` → resolves `_output_futures[async_output_id]` (with batch split via `_batch_split_map`)
-   - Non-async messages → `_sync_result_buffer` for other RPCs
-
-4. **collective_rpc() two-path dispatch** (`multiproc_executor.py`):
-   - **Path 1**: `execute_model` / `execute_model_batch` → generates `rpc_id`, registers Future, waits for pump to deliver `compute_done`
-   - **Path 2**: All other RPCs → `_sync_result_buffer` (pump-fed) or `_result_mq` (step-mode, no pump)
-
-5. **record_device_event** (`platforms/`): Cross-platform GPU event recording for side-stream synchronization. Implemented for CUDA, NPU, ROCm, XPU, MUSA; base class returns `None` (safe no-op).
-
-6. **IPC side-stream D2H** (`ipc.py`): `pack_diffusion_output_shm()` and helpers accept `d2h_stream` parameter; use `pin_memory` + `copy_(non_blocking=True)` on the side stream instead of synchronous `.cpu()`.
-
-### Batch Split
-
-When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently.
-
-## Model Coverage
-
-### Currently Supported (request-mode, `step_execution=False`)
-
-All models running in request-mode (`step_execution=False`, the default) automatically use async output.
-
-**Verified models**:
-
-| Model | Type | `supports_request_batch` |
-|---|---|---|
-| **HunyuanImage-3.0** | Image | `False` |
-| **Qwen-Image** | Image | `True` |
-
-Other models with `step_execution=False` are also supported but not yet verified.
-
-### Not Yet Supported (step-mode, `step_execution=True`)
-
-When `step_execution=True` (or `streaming_output=True`, which auto-enables step-mode), models use `execute_stepwise` instead of `execute_model`, which is not in the async Path 1 whitelist. Async output is not applicable.
-
-**Verified**: Helios (`step_execution=True`, does not benefit from this feature).
-
-Other models with `step_execution=True` are also not applicable.
-
-Adapting step-mode requires additional design because:
-
-- Each step's `RunnerOutput` must be available synchronously for the scheduler to advance
-- Intermediate latent tensors may not need D2H (only the final step needs D2H + postprocess)
-- Steps are serial (step N+1 depends on step N output), so the overlap benefit is limited to "final step's D2H overlaps with the next request's forward"
-
-**Planned approach**: first cover only the final step's async D2H, keep intermediate steps synchronous. This matches the request-mode benefit pattern.
-
-## Configuration
-
-No configuration needed. Async output is automatically enabled when `step_execution=False` (default).
-
-| `step_execution` | Mode | Async Output | Pump Thread | Worker BG Thread |
-|---|---|---|---|---|
-| `False` (default) | request-mode | ✅ enabled | ✅ started | ✅ started |
-| `True` | step-mode | ❌ disabled | ❌ not started | ❌ not started |
-
-## Related Files
-
-- `vllm_omni/diffusion/data.py`: `AsyncDiffusionOutput`, `AsyncOutputKind`, `DiffusionOutput.async_output_id`
-- `vllm_omni/diffusion/worker/diffusion_worker.py`: `WorkerProc._return_result()`, `_async_output_loop()`, `_generate_async_output_id()`
-- `vllm_omni/diffusion/executor/multiproc_executor.py`: `ResultPumpThread`, `collective_rpc()` two-path dispatch, `wait_output_ready()`, `_batch_split_map`
-- `vllm_omni/diffusion/diffusion_engine.py`: `step()` / `step_streaming()` / `add_req_and_wait_for_response()` async output waiting
-- `vllm_omni/diffusion/sched/request_scheduler.py`: `update_from_output()` — `async_output_id` → `FINISHED_COMPLETED`
-- `vllm_omni/diffusion/ipc.py`: `pack_diffusion_output_shm()` side-stream D2H path
-- `vllm_omni/platforms/interface.py`: `OmniPlatform.record_device_event()` base class
-- `vllm_omni/platforms/cuda/platform.py`: `CudaOmniPlatform.record_device_event()`
-- `vllm_omni/platforms/npu/platform.py`: `NPUOmniPlatform.record_device_event()`
-- `vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py`: `get_hunyuan_image3_post_process_func()`, postprocess removal
-- `vllm_omni/diffusion/registry.py`: `_DIFFUSION_POST_PROCESS_FUNCS` postprocess registry

docs/design/feature/diffusion_continuous_batching.md

@@ -1,232 +1,165 @@
-# Diffusion Continuous Batching
+# Continuous Batching for Step-Wise Diffusion
 
-This document describes the unified diffusion execution architecture, including
+!!! warning "Experimental Feature"
-request execution, request-level batching, step execution, continuous batching,
+    This feature is experimental. It currently applies only to native
-and output delivery. For user-facing configuration and CLI examples, see
+    diffusion pipelines running with `step_execution=True`.
-[Diffusion Execution Modes](../../user_guide/diffusion/execution_modes.md).
+
+This document describes the batching extension built on top of
+[Diffusion Step Execution](diffusion_step_execution.md). The base
+step-execution contract is unchanged. The batching work is mainly in the
+scheduler and runner layers.
+
+## Why It Helps
+
+Step-wise execution breaks a long denoise loop into scheduler-visible units.
+That gives the runtime a place to admit other compatible requests between
+steps instead of waiting for an entire request to finish.
+
+This matters most in low-MFU or bursty serving scenarios:
+
+- one request's denoise step may not fully saturate the GPU
+- several compatible requests can share the same denoise forward pass
+- throughput and device utilization can improve without changing request-local
+  scheduler state
+
+This is **not** a guaranteed single-request latency win. The main benefit is
+usually higher utilization and better throughput when the workload contains
+multiple in-flight compatible requests.
 
 ## Overview
 
-`DiffusionEngine` selects an execution mode from `step_execution` and uses
+With continuous batching enabled:
-`max_num_seqs` as that mode's scheduler capacity:
 
-| Configuration | Engine mode | Scheduler | Execution |
+- the scheduler may keep multiple compatible requests active at the same time
-|---|---|---|---|
+- the runner packs request-local step state into one `InputBatch`
-| `step_execution=False`, `max_num_seqs=1` | `REQUEST_BATCH` | `RequestScheduler` | One complete request-level `forward()` |
+- `denoise_step()` runs on that batch
-| `step_execution=False`, `max_num_seqs>1` | `REQUEST_BATCH` | `RequestScheduler` | One fused `forward()` over compatible requests |
+- `step_scheduler()` and `post_decode()` still run per request
-| `step_execution=True`, `max_num_seqs=1` | `STEP_BATCH` | `StepScheduler` | One request advanced one denoise step per scheduler tick |
-| `step_execution=True`, `max_num_seqs>1` | `STEP_BATCH` | `StepScheduler` | Compatible requests advanced together in step waves |
 
-Serial request execution is handled inside `REQUEST_BATCH`; it is not a
+The current implementation is conservative:
-separate engine mode. Similarly, continuous batching is the multi-request
-configuration of `STEP_BATCH`, not an independent execution mode.
 
-The engine performs the following initialization:
+- only compatible requests are batched together
+- per-request progress and completion remain independent
 
-1. Resolve model-specific pre- and post-processing hooks.
+Here, "continuous batching" means the step-wise path enabled by
-2. Select `REQUEST_BATCH` or `STEP_BATCH`.
+`step_execution=True`. Request-mode `DiffusionRequestBatch` is static
-3. Construct the configured `DiffusionExecutor`.
+request-level batching for one full pipeline `forward()` call; it does not
-4. Construct `RequestScheduler` or `StepScheduler`, unless a scheduler was
+admit or remove requests between denoise steps.
-   explicitly injected.
-5. Bind `execute_batch` or `execute_step`.
 
-`DiffusionEngine.make_engine()` owns startup warmup after engine construction.
+## Enablement
-This keeps construction, backend selection, and warmup failure cleanup in one
-lifecycle.
 
-## Unified Output Stream
+Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
+above `1` if you want batching:
 
-All asynchronous diffusion requests use the same per-request output stream,
+```bash
-regardless of whether the caller exposes streaming results to the user.
+vllm serve Qwen/Qwen-Image --omni \
-
+  --port 8091 \
-```python
+  --step-execution \
-async for output in engine.step_streaming(request):
+  --max-num-seqs 8
-    ...
 ```
 
-When a request is admitted, the engine creates an
+`--max-num-seqs 1` keeps the step-wise path without enabling batching.
-`asyncio.Queue[DiffusionOutput]` keyed by its request ID. The busy loop sends
-both intermediate chunks and final results through that queue:
 
-```text
+For a reproducible replay flow using the bundled serving benchmark, see the
-request
+Qwen-Image replay commands in
-  -> scheduler.add_request()
+[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
-  -> scheduler.schedule()
+and
-  -> execute_batch() or execute_step()
+[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
-  -> scheduler.update_from_output()
-  -> per-request output queue
-  -> step_streaming()
-```
 
-Streaming callers forward every yielded output. Non-streaming callers consume
+## Scheduler
-the same generator to completion and return only its final output. This avoids
-maintaining separate future-based and queue-based result paths.
 
-`DiffusionEngine.step()` and
+The scheduler derives its batch capacity from `max_num_seqs` through
-`async_add_req_and_wait_for_response()` remain deprecated compatibility
+`max_num_running_reqs`.
-wrappers. New integrations should consume `step_streaming()` or
-`async_add_req_and_stream_response()`.
 
-### Completion and Cancellation
+Batch admission is gated by
+[`StepBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
+which is built from shape-sensitive and CFG-sensitive sampling fields. This is
+the core correctness rule for batching: requests are only co-batched when they
+share the same denoise tensor contract.
 
-Output delivery and scheduler lifecycle are related but separate:
+There are three important details:
 
-- A terminal scheduler result always finalizes request state.
+- `num_inference_steps` is not part of the key, so requests with different
-- A disconnected or cancelled consumer removes only its delivery queue.
+  total step counts can still share a batch
-- The scheduler may still finish and clean up the request after its queue has
+- requests also do not need to be at the same current denoise progress; active
-  been removed.
+  requests can continue batching even when their current step indices diverge
-- Engine shutdown sends an error output to all remaining streams.
+- admission is still FIFO, so an incompatible request at the head of the
+  waiting queue blocks later compatible requests
 
-This separation prevents cancelled consumers from leaving active request IDs in
+Today that compatibility rule is still shape-sensitive. `height`, `width`,
-the scheduler.
+`num_frames`, and CFG-related fields remain part of the key, so different
+resolutions or incompatible guidance settings do **not** co-batch yet. The
+key also covers LoRA identity (`lora_int_id`, `lora_scale`), so requests
+targeting different adapters or scales run in separate batches and the
+worker can activate exactly one adapter per step.
 
-## Request-Batch Execution
+The scheduler batching unit is one logical `OmniDiffusionRequest`. In the
+step-wise path, runtime tensor batching is represented as `StepInputBatch`. For
+request-mode prompt semantics, see
+[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
 
-Request-batch execution runs the complete diffusion pipeline for each scheduler
+## Runner
-wave. Each `OmniDiffusionRequest` remains an independent logical request with
-its own prompt, sampling parameters, seed, request ID, output, error, and abort
-state.
 
-With `max_num_seqs=1`, `execute_batch()` handles the serial request path. With a
+The runner keeps persistent per-request execution state in
-larger value, a pipeline can opt into fused execution by declaring:
+[`StepRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
+while the scheduler owns a separate lightweight
+[`SchedulerRequestState`](gh-file:vllm_omni/diffusion/sched/interface.py)
+for queueing and lifecycle tracking.
 
-```python
+For each step, the runner builds an
-supports_request_batch = True
+[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
-```
+active request states:
 
-Its `forward()` method must accept `DiffusionRequestBatch` and return one
+- prompt embeddings and masks are normalized and padded
-`DiffusionOutput` per request. The runner validates the result count and maps
+- dynamic tensors such as `latents` and `timesteps` are gathered each step
-outputs back to their original request IDs with `BatchRunnerOutput`.
+- buffers are reused when batch composition stays the same
 
-Current in-tree request-batch implementations include the Flux, LTX-2, SD3, and
+The step-wise batched path is:
-Qwen-Image pipelines. Treat the pipeline capability flag in the source as
-authoritative because support continues to expand.
 
-### Request-Batch Data Flow
+1. Run `prepare_encode()` for newly admitted requests.
-
-- `DiffusionSchedulerOutput` carries scheduled request IDs and payloads.
-- `DiffusionRequestBatch` provides the pipeline-facing static batch.
-- `BatchRunnerOutput` carries per-request results back to the engine.
-
-Request-local setup, such as seed handling and output/error mapping, remains
-per request. Cache refresh, homogeneous LoRA activation, and
-`pipeline.forward(req_batch)` can be shared by the batch.
-
-### Compatibility and Admission
-
-`RequestBatchSamplingParamsKey` contains fields that affect the tensor contract,
-including shape, guidance, output count, and LoRA identity. Only compatible
-requests can share a fused forward call.
-
-Admission is FIFO and conservative. An incompatible request at the front of the
-waiting queue can prevent later compatible requests from joining the current
-wave.
-
-`request_batch_max_wait_ms` optionally delays the first schedule operation of a
-new wave so bursty arrivals can accumulate. It applies only when:
-
-- the pipeline supports request batching;
-- step execution is disabled;
-- no requests are currently running; and
-- the configured wait is greater than zero.
-
-The wait ends when the queue reaches capacity, stabilizes briefly, reaches its
-deadline, or the engine stops.
-
-## Step-Batch Execution
-
-Step execution exposes denoising progress to the scheduler. A supporting
-pipeline implements four stateful operations:
-
-| Operation | Responsibility |
-|---|---|
-| `prepare_encode(state)` | Validate input, encode prompts, initialize latents and timesteps, and create request-local scheduler state |
-| `denoise_step(input_batch, *, states=...)` | Run one denoise forward for the scheduler-provided request states |
-| `step_scheduler(state, noise_pred)` | Update latents and advance request progress |
-| `post_decode(state)` | Decode and postprocess a completed request or output boundary |
-
-Persistent request state lives in `StepRequestState`. Pipeline-specific fields
-that do not belong in the shared contract should be stored in `state.extra`.
-Queueing and lifecycle metadata remain in the scheduler's request state.
-
-Current native pipelines that explicitly enable step execution include
-Qwen-Image, HunyuanImage3, and Helios. Step execution alone does not imply
-continuous-batching support: Qwen-Image accepts batched step states.
-HunyuanImage3 accepts batched step states only when its resolved self-attention
-backend is `TORCH_SDPA`; otherwise it rejects groups larger than one request.
-Configure `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or
-`diffusion_attention_config.default.backend=TORCH_SDPA` when
-`max_num_seqs>1`. See the
-[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
-for its validated configuration. Helios supports only a single active step
-request and must use `max_num_seqs=1`.
-
-### Continuous Batching
-
-When `max_num_seqs>1`, `StepScheduler` can keep multiple compatible requests
-active. The runner gathers their state into `InputBatch`, performs one batched
-denoise forward, then applies scheduler updates per request:
-
-1. Receive transferred KV payloads, then run `prepare_encode()` for newly
-   admitted requests.
 2. Build or refresh `InputBatch`.
-3. Run one batched `denoise_step(input_batch, states=states)`.
+3. Run one batched `denoise_step(input_batch)`.
-4. Slice noise predictions back to each request.
+4. Slice the batched `noise_pred` back per request.
-5. Run each request's `step_scheduler()`.
+5. Run per-request `step_scheduler()`.
-6. Run `post_decode()` at a chunk boundary or request completion.
+6. Run `post_decode()` only for requests that finished denoising.
-7. Scatter updated latents back into persistent request state.
+7. Scatter updated latents back into persistent request state with
+   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
 
-`StepBatchSamplingParamsKey` protects the batched tensor contract. Requests can
+This keeps the shared work limited to the denoise forward pass while preserving
-have different total step counts and current step indices, but shape-sensitive,
+request-local scheduler state and outputs.
-CFG-sensitive, and LoRA fields must be compatible. FIFO head-of-line blocking
-also applies here.
 
-Chunk-capable pipelines may emit intermediate results. Final-only step
+## Engine
-pipelines emit when the request finishes even if `streaming_output=True`.
 
-## Model Author Guidelines
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
+the background loop and async add-request path needed for multiple requests to
+accumulate in the scheduler.
 
-When converting a request-level pipeline:
+When `step_execution=True`, the engine routes work through the step-wise
+executor path. The continuous batching behavior is defined by scheduler-side
+compatibility gating and runner-side `StepInputBatch` packing.
 
-- Reuse the same helpers as `forward()` to avoid behavior drift.
+## Current Limitations
-- Copy request-scoped scheduler state into `state.scheduler`.
-- Advance `state.step_index` only in `step_scheduler()`.
-- Keep model forward work in `denoise_step()`.
-- Keep latent mutation in `step_scheduler()`.
-- Keep final decoding equivalent to the tail of `forward()`.
-- Store masks, condition latents, and other request-local tensors in the
-  request state rather than on the shared pipeline.
 
-Before setting `supports_step_execution = True`, validate output parity for the
+- Experimental feature; use `max_num_seqs=1` for the conservative single-request
-same seed and sampling parameters, scheduler isolation across concurrent
+  step path.
-requests, abort cleanup, progress reporting, and all supported CFG paths.
+- Only native pipelines that already support `step_execution=True`.
+- Only homogeneous batches keyed by `StepBatchSamplingParamsKey` are supported.
+- `cache_backend`, KV transfer, and other request-mode extras are not wired
+  into the batched step-wise path yet.
+- Future work can relax the current same-shape restriction with richer
+  heterogeneous batching policies such as bucketing or padded execution for
+  different resolutions.
 
-Before setting `supports_request_batch = True`, validate the batch input
+## Related Files
-contract, one-output-per-request result shape, request identity and error
-mapping, seeded concurrency, LoRA compatibility, and tensor IPC.
 
-## Limitations
+- Scheduler base:
-
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
-- Request batching requires an explicit pipeline capability when
+- Scheduler interface:
-  `max_num_seqs>1`.
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Step execution requires the four-method stateful pipeline contract.
+- Step scheduler:
-- Both batching paths currently require homogeneous compatibility keys.
+  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
-- FIFO scheduling can reduce batching opportunities.
+- Runner:
-- `request_batch_max_wait_ms` trades first-request latency for burst
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-  coalescing.
+- Input batch:
-- All diffusion cache backends are currently unsupported in step mode.
+  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
-- KV transfer is supported for newly admitted step requests; some other
+- Tests:
-  request-mode extras remain unsupported.
+  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)
-- Step continuous batching remains experimental; use `max_num_seqs=1` for the
-  conservative step path.
-
-## Related Implementation
-
-- Engine: [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
-- Model contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
-- Scheduler interface: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Request scheduler: [`vllm_omni/diffusion/sched/request_scheduler.py`](gh-file:vllm_omni/diffusion/sched/request_scheduler.py)
-- Step scheduler: [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
-- Runner: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-- Request batch: [`vllm_omni/diffusion/worker/request_batch.py`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
-- Step input batch: [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)

docs/design/feature/diffusion_request_level_batching.md

@@ -0,0 +1,175 @@
+# Request-Level Batching for Diffusion
+
+This document describes the request-mode batching path for diffusion pipelines.
+For end-user enablement and tuning, see
+[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
+
+This is separate from
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+Request-level batching runs one full pipeline `forward()` over a static batch of
+compatible requests. Step-wise continuous batching admits work between denoise
+steps when `step_execution=True`.
+
+## Why It Helps
+
+The request-level design avoids coupling several logical requests to one request
+object. This keeps request identity, abort/error handling, and per-request
+metadata unambiguous while still allowing one fused pipeline forward pass for
+bursty or concurrent traffic.
+
+## Overview
+
+With request-level batching enabled:
+
+- each `OmniDiffusionRequest` contains one `prompt` and one `request_id`
+- the scheduler groups compatible waiting requests into one scheduler wave
+- `DiffusionRequestBatch` wraps the scheduled requests for pipeline `forward()`
+- batch-capable pipelines return `list[DiffusionOutput]`, one output per
+  request
+- `BatchRunnerOutput` maps each result back to its original `request_id`
+
+Pipelines opt in with `supports_request_batch = True` and a `forward()` method
+that accepts `DiffusionRequestBatch` and returns `list[DiffusionOutput]`.
+Pipelines that do not opt in keep the existing per-request execution path.
+
+## Enablement
+
+Request-level batching is the request-mode path, so `step_execution` must remain
+disabled. Increase `max_num_seqs` above `1` to let the scheduler keep multiple
+compatible requests active:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4
+```
+
+For bursty online ingress, `request_batch_max_wait_ms` can add a bounded
+admission wait before the first `schedule()` of a scheduler wave:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+`request_batch_max_wait_ms=0` disables this wait and is the default.
+
+## Request Contract
+
+`OmniDiffusionRequest` represents one logical request. It owns one prompt,
+sampling parameters, request id, and request-local metadata. Runtime batches are
+formed by the scheduler and represented separately from the request payload.
+
+Runtime batching is represented by:
+
+- [`DiffusionSchedulerOutput`](gh-file:vllm_omni/diffusion/sched/interface.py)
+  for scheduled request ids and request payloads
+- [`DiffusionRequestBatch`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
+  for the pipeline-facing request batch
+- [`BatchRunnerOutput`](gh-file:vllm_omni/diffusion/worker/utils.py) for
+  per-request results
+
+`DiffusionRequestBatch` intentionally exposes compatibility properties such as
+`prompts`, `sampling_params`, `request_id`, and `kv_sender_info` so migrated
+pipelines can stay close to upstream code while using a batch-aware contract.
+
+## Scheduler
+
+The scheduler derives its capacity from `max_num_seqs` through
+`max_num_running_reqs`. It exposes waiting/running queue counters so the engine
+can decide whether admission wait is useful before scheduling a new wave.
+
+Batch compatibility is controlled by
+[`RequestBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key
+contains shape-sensitive and guidance-sensitive fields, including output count
+and LoRA identity. Requests with incompatible shapes, CFG settings, output
+counts, LoRA adapters, or LoRA scales are kept in separate batches.
+
+Admission is conservative:
+
+- the scheduler only batches compatible requests
+- FIFO ordering is preserved
+- an incompatible request at the head of the waiting queue blocks later
+  compatible requests
+
+## Engine
+
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) resolves
+request-batch capability during initialization from the configured pipeline
+class, including custom pipeline classes.
+
+The capability check uses the pipeline class attribute
+`supports_request_batch = True`. Pipelines that set this attribute must implement
+a request-batch-compatible `forward()` contract and return one
+`DiffusionOutput` per request; the runner validates that return shape at runtime.
+
+When the selected pipeline is batch-capable and `step_execution=False`, request
+mode routes scheduler waves through the batch executor path. Otherwise it keeps
+the per-request executor path.
+
+The optional admission wait runs only when:
+
+- request batching is supported
+- `step_execution=False`
+- `request_batch_max_wait_ms > 0`
+- no requests are currently running
+
+The wait exits early when the waiting queue reaches capacity, when the queue is
+stable for a short window, when the deadline expires, or when the engine stops.
+
+## Executor And Runner
+
+The executor exposes two request-mode entries:
+
+- `execute_request`: one worker call per scheduled request
+- `execute_batch`: one worker call for the whole `DiffusionSchedulerOutput`
+
+On the batch path, the worker builds a `DiffusionRequestBatch` and runs the
+pipeline once. Request-local setup remains per request:
+
+- KV transfer metadata
+- random generator and seed handling
+- request output/error/abort mapping
+
+Shared batch setup happens once per batch when possible:
+
+- cache refresh
+- LoRA activation for the homogeneous adapter key
+- pipeline `forward(req_batch)`
+
+Large tensor IPC still uses the shared-memory packing path. The packer traverses
+both normal `RunnerOutput.result` wrappers and nested batch results so batched
+outputs do not fall back to pickle IPC for tensor payloads.
+
+## Current Limitations
+
+- Only pipelines that declare the request-batch contract use fused batch
+  execution.
+- Batches are homogeneous under `RequestBatchSamplingParamsKey`; heterogeneous resolution or
+  incompatible guidance settings do not co-batch yet.
+- FIFO scheduling can reduce batching opportunities when an incompatible
+  request is at the front of the queue.
+- `request_batch_max_wait_ms` improves burst coalescing but can add latency to
+  the first request in a scheduler wave. Keep it small for latency-sensitive
+  serving.
+- Step-wise continuous batching is documented separately and only applies when
+  `step_execution=True`.
+
+## Related Files
+
+- Request object and request batch:
+  [`vllm_omni/diffusion/request.py`](gh-file:vllm_omni/diffusion/request.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Engine:
+  [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
+- Worker runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Executor interface:
+  [`vllm_omni/diffusion/executor/abstract.py`](gh-file:vllm_omni/diffusion/executor/abstract.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_engine.py`](gh-file:tests/diffusion/test_diffusion_engine.py)

docs/design/feature/diffusion_step_execution.md

@@ -0,0 +1,131 @@
+# Adding Step Execution Support for Diffusion Pipelines
+
+This guide documents vLLM-Omni's stepwise diffusion contract for model authors
+and contributors implementing `step_execution=True` support for a diffusion
+pipeline.
+
+For end-user enablement, supported models, and current limitations, see
+[Step Execution](../../user_guide/diffusion/step_execution.md).
+
+This document describes the base step-execution contract only. For the
+experimental batching policy layered on top of the step-wise path, see
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+
+## Current Support Scope
+
+`step_execution` is **not** a generic diffusion toggle. It only works for
+pipelines that implement the segmented stateful contract in
+[`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
+
+This page is intentionally author-facing. Treat runtime enablement
+(`step_execution=True` in Python or `--step-execution` in serving) as an
+opt-in user knob layered on top of the implementation contract below.
+
+Current in-tree support:
+
+| Pipeline | Example models | Step execution |
+|----------|----------------|----------------|
+| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
+| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
+
+Current engine/runtime limitations:
+
+- Continuous batching with `max_num_seqs > 1` is experimental and documented in
+  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+  Keep `max_num_seqs=1` for the conservative single-request step path.
+- `cache_backend` is not supported in step mode.
+- Request-mode extras such as KV transfer are not wired into step mode yet.
+- Unsupported pipelines now fail early during model loading instead of failing on the first request.
+
+## Execution Contract
+
+Step mode is driven by four pipeline methods plus the shared mutable request
+state object:
+
+- `prepare_encode(state)`: one-time request preparation.
+- `denoise_step(state)`: compute the noise prediction for the current step.
+- `step_scheduler(state, noise_pred)`: mutate latents and advance step state.
+- `post_decode(state)`: decode the final output after denoising is complete.
+
+The state lives in
+[`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
+as `StepRequestState`. Store request-scoped tensors there, or use
+`state.extra` for model-specific fields that do not justify extending the core
+dataclass.
+
+The worker-side step loop lives in
+[`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py):
+
+1. `prepare_encode()` runs once for a new request.
+2. `denoise_step()` runs every scheduler tick.
+3. `step_scheduler()` mutates `state.latents` and advances `state.step_index`.
+4. `post_decode()` runs exactly once after `state.denoise_completed` becomes true.
+
+## Recommended Split
+
+When converting an existing request-level `forward()` pipeline, keep the split
+strict and mechanical:
+
+| Request-level phase | Stepwise method | What belongs there |
+|---------------------|-----------------|--------------------|
+| Input validation, prompt encoding, latent init, timestep prep, per-request scheduler creation | `prepare_encode()` | Anything that should happen once per request |
+| Transformer forward / noise prediction | `denoise_step()` | Pure denoise computation for the current timestep |
+| `scheduler.step(...)` and `step_index += 1` | `step_scheduler()` | Only latent/state mutation for one step |
+| VAE decode / postprocess | `post_decode()` | Final decode only |
+
+Keep the stepwise path reusing the same helpers as the request-level path
+whenever possible. Reimplementing the denoise loop from scratch is the easiest
+way to introduce behavioral drift.
+
+## Qwen-Image Reference
+
+[`pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)
+is the reference implementation and is split correctly for the current
+contract:
+
+- `prepare_encode()` reuses `_prepare_generation_context()` so prompt encoding,
+  latent init, timestep creation, CFG setup, and shape bookkeeping stay aligned
+  with `forward()`.
+- `prepare_encode()` deep-copies `self.scheduler` **after**
+  `prepare_timesteps()` so request-specific scheduler state is isolated.
+- `denoise_step()` reuses `_build_denoise_kwargs()` plus
+  `predict_noise_maybe_with_cfg()`, so sequential CFG, CFG-parallel, and
+  non-CFG behavior stay identical to the request-level path.
+- `step_scheduler()` only calls
+  `scheduler_step_maybe_with_cfg(..., per_request_scheduler=state.scheduler)`
+  and increments `state.step_index`.
+- `post_decode()` reuses `_decode_latents()`, so the final image decode matches
+  the normal `forward()` path.
+
+That decomposition is the target pattern for future models.
+
+## Rules For New Pipelines
+
+- Do not keep request-scoped scheduler state on `self.scheduler`. Copy it into
+  `state.scheduler` during `prepare_encode()`.
+- Do not mutate `state.step_index` inside `denoise_step()`. Only
+  `step_scheduler()` should advance the step.
+- Do not decode partial outputs in `denoise_step()` or `step_scheduler()`.
+- If the request-level pipeline has condition latents, masks, or edit-specific
+  tensors, store them in `state` or `state.extra`, not in global pipeline
+  attributes.
+- Preserve CFG behavior by sharing the same helper path used by `forward()`.
+- Keep `post_decode()` equivalent to the tail of `forward()`.
+
+## Validation Checklist
+
+Before marking a pipeline as `supports_step_execution = True`, verify:
+
+- Stepwise output matches request-level output for the same seed and sampling params.
+- Per-request scheduler state is isolated across concurrent requests.
+- Abort during denoise does not leak cached state.
+- `step_index` reported by `RunnerOutput` matches the scheduler progress.
+- CFG-parallel and non-CFG paths both work if the request-level pipeline supports them.
+
+## Related Files
+
+- Contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
+- State: [`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
+- Runner loop: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Scheduler transport: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Reference pipeline: [`vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)

docs/design/feature/distributed_layerwise_offload.md

@@ -1,156 +0,0 @@
-# Distributed Layerwise Offload
-
-This document describes distributed layerwise offload (DLO) for diffusion
-models. DLO keeps only a small number of DiT blocks on the accelerator and
-streams the remaining blocks from host memory. The distributed backend can
-either shard those host-side weights across an existing parallel group or keep
-the standard loader's rank-local weights and avoid an additional collective.
-
-For user-facing commands, see the
-[distributed layerwise offloading guide](../../user_guide/diffusion/cpu_offload.md)
-and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md).
-
-## Status
-
-DLO is implemented for multi-device diffusion execution. The default
-AllGather path is the primary path for DP and SP deployments. The
-`--dlo-no-use-allgather` path is a rank-local compatibility mode: it is useful
-for standard-loader sharding, workstation bring-up, and systems where an
-additional DLO collective is undesirable, but it does not reduce host weight
-storage across ranks.
-
-The compatibility matrix below describes the current implementation. The
-unit-level guards are covered, but not every parallelism combination has a
-full model-and-hardware end-to-end test.
-
-## Design
-
-### DLO consumes the existing parallel topology
-
-DLO does not create a new DP, TP, or SP topology. It reads the configured
-`DiffusionParallelConfig` and attaches offload hooks to the DiT blocks after the
-standard distributed groups have been initialized.
-
-The DLO weight-sharding group is selected as follows:
-
-1. Use the existing DP group when `data_parallel_size > 1`.
-2. When DP is one and SP is greater than one, use the SP group.
-3. Otherwise, run rank-locally without a DLO process group.
-
-TP is deliberately not used as DLO's AllGather group. HSDP has its own
-parameter-sharding lifecycle and is not allowed to be sharded a second time by
-DLO's AllGather path.
-
-### AllGather path
-
-With the default `dlo_use_allgather=True`, each rank stores approximately
-`1 / group_size` of each streamable block in pinned host memory. The next
-block's shard is copied to a device buffer and reconstructed with
-`all_gather_into_tensor` on a communication stream while the current block is
-executing.
-
-```text
-Compute:    [Block N]             [Block N+1]          [Block N+2]
-H2D:                      [shard N+1]           [shard N+2]
-AllGather:                [full N+1]             [full N+2]
-Buffers:    [current slot]       [prefetch slot]       [current slot]
-```
-
-![DLO double-buffer prefetch pipeline](../figures/dlo/dlo_pipeline.gif)
-
-The backend uses two shared device buffers, so accelerator weight residency is
-bounded by the largest streamed blocks rather than the complete model.
-
-When DP is greater than one, the engine can process one request per DP rank in
-the same denoising wave. Because AllGather is a collective, all participating
-requests must take the same execution path at every denoising step.
-
-### Rank-local path without DLO AllGather
-
-With `--dlo-no-use-allgather`, DLO forces its internal offload shard size to
-one. The regular model loader remains responsible for preparing each rank's
-weights, including TP-local tensors or HSDP-managed parameters. DLO then
-streams those rank-local tensors block by block using H2D copies only.
-
-This mode means:
-
-- DP still provides independent replicas, but DLO does not shard weights
-  across DP ranks.
-- SP still performs its normal activation/attention collectives, but DLO does
-  not shard weights across SP ranks.
-- TP/HSDP/SP collectives, if configured, are not disabled by this flag; only
-  DLO's additional weight AllGather is disabled.
-- Pure DP deployments keep a full host-side model copy per rank, subject to
-  shared OS page-cache behavior.
-- The scheduler does not require a synchronized DP request wave for DLO.
-
-This path is intentionally not implemented by reusing the AllGather mmap
-loader. It relies on the standard loader so model-specific TP/HSDP transforms
-remain intact.
-
-## Parallelism compatibility
-
-| Parallelism | DLO + AllGather | DLO without AllGather |
-|---|---|---|
-| **DP** | Supported primary path. DLO shards host weights across the DP group and can run DP multi-concurrency. | Supported rank-local path. DP replicas remain independent; no DLO weight sharding or cross-DP DLO collective. |
-| **SP** | Supported in the implementation. With DP=1, DLO uses the SP group for host-weight sharding; SP still shards sequence/activation work. | SP remains active, but DLO keeps standard-loader rank-local weights and adds no SP weight collective. |
-| **TP > 1** | Unsupported in the DLO mmap path. TP-aware loader callbacks are bypassed, so the backend rejects this configuration when it enters that path. | Standard loading is retained and TP-local tensors can be streamed. This is the intended compatibility path, but it still needs broader model and hardware validation. |
-| **HSDP** | Rejected. HSDP has already sharded parameters, so DLO AllGather would double-shard them. | Accepted by configuration. HSDP owns parameter sharding and its own gathers; DLO only stages rank-local parameters. End-to-end coverage is limited. |
-
-### Combined dimensions
-
-- **DP + SP:** DLO uses the DP group for weight sharding when DP is greater
-  than one; SP continues to use its own sequence-parallel group. If DP is one,
-  the SP group becomes DLO's sharding group in AllGather mode.
-- **DP + TP/SP without AllGather:** standard model loading defines the
-  rank-local tensor layout. DLO adds no cross-DP, cross-TP, or cross-SP weight
-  collective.
-- **HSDP + SP:** the general parallel configuration permits HSDP over SP, but
-  DLO must use `--dlo-no-use-allgather`. HSDP remains responsible for weight
-  materialization and synchronization.
-- **HSDP + DP or TP:** rejected independently by the diffusion parallel
-  configuration.
-
-## Request and loading constraints
-
-AllGather DP multi-concurrency requires:
-
-- explicit `num_inference_steps`;
-- the same `num_inference_steps` for all requests in a wave; and
-- identical request arguments that affect the collective execution path.
-
-The no-AllGather path does not impose these DLO-specific synchronized-wave
-requirements.
-
-The mmap loader is used only by the supported DLO+AllGather path when the
-model has a compatible checkpoint layout. Online quantization is incompatible
-with that sharded mmap path; use `--dlo-no-use-allgather` or disable online
-quantization. Models that do not meet the mmap requirements use the regular
-loader path.
-
-## Validation coverage
-
-Current source-level validation includes:
-
-- HSDP + DLO + AllGather rejection;
-- HSDP + DLO without AllGather acceptance at configuration level;
-- TP rejection in the DLO+AllGather mmap path;
-- resident-layer requests requiring no-AllGather;
-- DP request-wave validation for denoising-step compatibility;
-- sharding, double-buffer, AllGather-size, and heterogeneous-block regression
-  tests.
-
-The highest-value missing coverage is end-to-end numerical comparison against
-ordinary layerwise offload for DP+SP, TP+no-AllGather, and HSDP+SP+no-AllGather
-on the target CUDA/NCCL or CANN/HCCL hardware.
-
-## Recommendations
-
-- Use **DP + DLO AllGather** for the supported throughput and host-memory
-  scaling path.
-- Use **SP + DLO AllGather** for long-sequence workloads when DP concurrency is
-  not the goal.
-- Use **no-AllGather** to bring up TP or workstation PCIe configurations, with
-  the expectation of higher host-memory use and lower validation confidence.
-- Prefer **HSDP alone** for production HSDP deployments until the combined
-  HSDP + DLO no-AllGather path has broader end-to-end coverage.

docs/design/feature/omni_async_output_materialization.md

@@ -1,386 +0,0 @@
-# Async Omni Output Materialization
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Performance](#performance)
-3. [Architecture](#architecture)
-4. [Enabling the Feature](#enabling-the-feature)
-5. [Compatibility and Fallbacks](#compatibility-and-fallbacks)
-6. [Related Files](#related-files)
-
-## Overview
-
-Async Omni output materialization moves CPU-side construction of
-`OmniModelRunnerOutput` out of the autoregressive (AR) decode critical path.
-It complements [Async Chunk](async_chunk.md): async chunk pipelines partial
-outputs between stages, while async output materialization prevents the
-construction of those partial outputs from blocking the next decode step.
-
-An AR stage must return sampled token IDs quickly because the scheduler needs
-them for the next decode step. However, an Omni stage can also produce hidden
-states, multimodal tensors, streaming inter-stage payloads, and connector
-metadata. Building these payloads can require device-to-host (D2H) copies,
-tensor slicing, flattening, and Python object construction.
-
-Without this feature, all of that work runs inline in `sample_tokens()`:
-
-```text
-sample tokens
-  -> update decode state
-  -> copy hidden and multimodal outputs to CPU
-  -> build per-request payloads
-  -> build the streaming wire payload
-  -> collect connector signals
-  -> return sampled tokens
-  -> launch the next decode step
-```
-
-With async output materialization, only work required by the next decode step
-stays on the critical path:
-
-```text
-sample tokens
-  -> update decode state
-  -> snapshot output state and start asynchronous D2H copies
-  -> register sampled tokens
-  -> launch the next decode step
-
-background output path
-  -> wait for the payload snapshot
-  -> build per-request payloads
-  -> build the streaming wire payload
-  -> collect connector signals
-  -> construct OmniModelRunnerOutput
-```
-
-This allows payload construction for step `N` to overlap with GPU decode work
-for step `N + 1`. It does not change model computation or generated output.
-
-!!! note "Platform and validation scope"
-    The performance and correctness validation in this document covers CUDA
-    and ROCm. This is a validation scope, not a runtime platform guard: XPU
-    inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so those
-    backends may enter the asynchronous path when its other guards pass. XPU
-    and MUSA have not been validated for this optimization. Ascend NPU uses a
-    separate `NPUARModelRunner` path that still constructs
-    `OmniModelRunnerOutput` and drains connector output synchronously before
-    wrapping the result as an asynchronous runner output.
-
-<p align="center">
-  <img
-    alt="Decode step gap before and after async output materialization"
-    src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"
-    width="100%"
-  >
-</p>
-
-<p align="center">
-  <em>
-    Figure 1: Moving payload construction off the decode path reduces the
-    observed gap between consecutive Talker steps from about 2.8 ms to 41 µs.
-    Adapted from the
-    <a href="https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization">
-      Qwen3-Omni optimization blog
-    </a>.
-  </em>
-</p>
-
-For Qwen3-Omni, the feature is enabled for both AR stages:
-
-- **Thinker**: Defers hidden-state and multimodal payload materialization.
-- **Talker**: Runs its decode-state postprocess eagerly, then defers codec
-  payload materialization. Because Code2Wav consumes codec codes rather than
-  Talker hidden states, the Talker also skips unnecessary hidden-state D2H.
-- **Code2Wav**: Does not use this path because it is a generation stage rather
-  than an AR stage.
-
-Qwen3-TTS uses the same mechanism in its AR Talker stage. The Talker updates
-the decode state it needs for the next step before returning, then constructs
-the codec payload for Code2Wav in the background.
-
-## Performance
-
-In the controlled Qwen3-Omni optimization sweep, enabling async output
-materialization on top of CUDA Graph and async chunk produced the following
-results at concurrency 64:
-
-| Configuration | Request throughput | Mean audio TTFP | Mean audio RTF |
-|---|---:|---:|---:|
-| CUDA Graph + async chunk | 9.3 req/s | 655 ms | 0.63 |
-| + async output materialization | 11.3 req/s | 631 ms | 0.47 |
-| Change | **+22%** | **-4%** | **-25%** |
-
-The sweep used `Qwen/Qwen3-Omni-30B-A3B-Instruct`, 640 Seed-TTS English
-prompts, and one replica each for Thinker, Talker, and Code2Wav. Async output
-materialization recovers throughput by removing CPU payload construction from
-the decode critical path without regressing time to first audio packet (TTFP).
-See the
-[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)
-for the complete benchmark methodology and results.
-
-The implementation was also validated against the synchronous safe-copy path.
-The text and audio output hashes matched, confirming that overlap does not
-change the generated result. See
-[PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476) for the
-correctness and profiling details.
-
-## Architecture
-
-### Output Lifecycle
-
-`OmniAsyncGPUModelRunnerOutput` extends vLLM's
-`AsyncGPUModelRunnerOutput`. It preserves the existing asynchronous
-sampled-token feedback while adding a background builder for the complete Omni
-output.
-
-The output lifecycle is:
-
-1. `GPUARModelRunner.sample_tokens()` samples tokens and performs the
-   bookkeeping required by the next decode step.
-2. The runner snapshots step-local metadata such as request IDs, token spans,
-   scheduler output, and query start locations.
-3. CUDA payload tensors are cloned before reusable model or CUDA Graph buffers
-   can be overwritten.
-4. The cloned payload is copied to pinned CPU memory on a dedicated CUDA
-   stream. A CUDA event records when the copy is ready.
-5. `OmniAsyncGPUModelRunnerOutput` starts a background thread that waits for
-   the payload event and builds `OmniModelRunnerOutput`.
-6. The AR runner immediately registers the sampled-token CPU copy with the
-   input batch and returns, allowing async scheduling to advance.
-7. When the engine calls `get_output()`, it joins the background builder,
-   propagates any builder exception, and finalizes sampled tokens and logprobs
-   through the upstream output implementation.
-
-```text
-GPU / decode path                         Background output path
-
-forward + sample
-      |
-      +-- clone output tensors
-      +-- enqueue D2H copy --------------> wait for D2H event
-      +-- register sampled tokens               |
-      +-- return async output                    +-- slice per request
-      |                                         +-- build multimodal payloads
-next decode step                                +-- partition streaming payloads
-                                                +-- build the wire payload
-                                                +-- drain connector signals
-                                                +-- build OmniModelRunnerOutput
-```
-
-### Safe State Snapshots
-
-The next scheduler step can mutate runner state while the background builder is
-still active. Most step-local builder inputs are therefore detached snapshots.
-The snapshotted state includes:
-
-- scheduler token counts and speculative-token metadata
-- request IDs and request-to-batch-index mappings
-- sampled token IDs, logprobs, and prompt logprobs
-- query start locations and scheduled-token spans
-- hidden states and multimodal output tensors
-- KV-connector and encoder-cache outputs captured for the step
-
-CUDA tensor cloning is required because CUDA Graph and model output buffers can
-be reused by the next forward pass. The dedicated copy stream and pinned host
-buffers make the D2H transfer asynchronous; the snapshot retains its cloned
-CUDA sources until the transfer event completes.
-
-### Live Runner State and Ownership
-
-The complete builder input is not a detached snapshot. Connector output state
-remains runner-owned: `get_omni_connector_output()` drains live, per-cycle
-connector signals after the streaming payload is built. Correctness therefore
-depends on that builder being the only consumer that drains connector signals
-for its output cycle. Receive-side connector state written by the background
-receiver is coordinated by the connector mixin's `_lock`.
-`OmniAsyncGPUModelRunnerOutput.get_output()` joins the builder and is the
-completion and exception boundary before the materialized output is consumed.
-
-The shared output builder also contains a full-payload accumulation branch
-that reads `self.requests`. That branch is not reachable when async Omni output
-materialization is enabled: this feature requires `async_chunk`, while
-`should_accumulate_full_payload_output()` returns `False` whenever
-`async_chunk` is enabled. The feature path partitions each step's payload into
-streaming inter-stage and client handoffs instead.
-
-### Output Builder
-
-The background builder performs the work that previously ran inline:
-
-- resolves which requests need downstream payloads
-- converts hidden and multimodal tensors into per-request CPU payloads
-- applies model-specific payload processing
-- partitions per-step payloads into inter-stage and client streams
-- creates the tensor-only `multimodal_outputs` wire payload
-- drains live connector readiness signals after building the streaming payload
-- constructs the final `OmniModelRunnerOutput`
-
-Input-side connector operations remain synchronous with model execution. In
-particular, receiving inputs and flushing previously completed connector
-outputs are not deferred because they affect scheduler-visible request state.
-
-### Qwen3-Omni Stage Behavior
-
-| Stage | Async output behavior | Reason |
-|---|---|---|
-| Thinker | Snapshots hidden states and multimodal outputs; builds the downstream payload in the background | The Talker needs the payload, but the next Thinker decode step only needs sampled-token feedback |
-| Talker | Runs lightweight postprocess eagerly; snapshots codec outputs; omits hidden states from the downstream payload | `hidden_states.last` is needed by the next Talker step, while Code2Wav only needs codec codes |
-| Code2Wav | Uses the normal generation-stage output path | Code2Wav is not executed by `GPUARModelRunner` |
-
-The Qwen3-TTS Talker follows the same pattern as the Qwen3-Omni Talker:
-postprocess runs eagerly because the next decode step needs its updated state,
-while codec payload construction runs asynchronously. Its Code2Wav stage also
-uses the normal generation-stage path.
-
-## Enabling the Feature
-
-There is no separate `async_omni_output` command-line or YAML option. The
-feature is selected automatically for model stages that opt in when the runtime
-conditions are safe. On the validated CUDA and ROCm configurations, using the
-bundled deployment profile is enough to turn it on for the supported
-Qwen3-Omni and Qwen3-TTS pipelines. The selection logic has no CUDA/ROCm-only
-guard; see [Compatibility and Fallbacks](#compatibility-and-fallbacks) for the
-other platform paths. The same profile does not enable background Omni
-materialization on Ascend NPU.
-
-### Example 1: Qwen3-Omni
-
-Start the full Qwen3-Omni pipeline:
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --omni \
-  --port 8091
-```
-
-The model registry automatically loads
-`vllm_omni/deploy/qwen3_omni_moe.yaml`. Its relevant settings are:
-
-```yaml
-async_chunk: true
-
-stages:
-  - stage_id: 0  # Thinker
-    enable_prefix_caching: false
-    # async_scheduling defaults to true for AR stages
-
-  - stage_id: 1  # Talker
-    enable_prefix_caching: false
-    # async_scheduling defaults to true for AR stages
-
-  - stage_id: 2  # Code2Wav
-    enable_prefix_caching: false
-    async_scheduling: false
-```
-
-With this profile, async output materialization activates automatically for:
-
-- **Stage 0, Thinker**: Defers hidden-state and multimodal payload
-  construction.
-- **Stage 1, Talker**: Defers codec payload construction after eagerly
-  updating its decode state.
-
-Stage 2 does not use this feature because Code2Wav is not an AR stage.
-
-### Example 2: Qwen3-TTS
-
-Start Qwen3-TTS with any checkpoint that uses the Qwen3-TTS Talker pipeline.
-For example:
-
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-  --omni \
-  --port 8091
-```
-
-The model registry automatically loads `vllm_omni/deploy/qwen3_tts.yaml`. Its
-relevant settings are:
-
-```yaml
-async_chunk: true
-
-stages:
-  - stage_id: 0  # Talker
-    async_scheduling: true
-    enable_prefix_caching: false
-
-  - stage_id: 1  # Code2Wav
-    enable_prefix_caching: false
-```
-
-Async output materialization activates only for Stage 0, the AR Talker. The
-Talker keeps the state required by its next decode step on the GPU, skips an
-unnecessary hidden-state D2H copy, and builds the codec payload for Code2Wav in
-the background. Stage 1 uses the generation-stage output path.
-
-The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
-because they use the same Talker implementation.
-
-!!! warning
-    Do not pass `--no-async-chunk` or enable prefix caching when you want this
-    optimization. Either change causes the runner to fall back to synchronous
-    output construction. Enabling async chunk alone cannot activate the feature
-    for a model that has not opted into async Omni output.
-
-!!! note "Prefix cache compatibility"
-    Async Omni output materialization and Omni prefix caching cannot currently
-    run together. `_should_use_async_omni_output()` returns `False` whenever
-    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
-    output materialization. Supporting both features requires snapshotting or
-    otherwise synchronizing prefix-cache merge and update state before the
-    background output path can consume it safely.
-
-## Compatibility and Fallbacks
-
-`GPUARModelRunner` uses the async output path only when all of the following
-runtime conditions hold:
-
-| Requirement | Reason |
-|---|---|
-| AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
-| `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
-| The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
-| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
-| Speculative decoding is disabled | Speculative output state is not included in this deferred path |
-| Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
-| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
-
-These checks are evaluated per stage on runners based on `GPUARModelRunner`.
-An unsupported combination does not prevent serving; it only falls back to
-synchronous output construction for that stage. CUDA and ROCm are the validated
-platforms. XPU inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so
-they may select the async path when these checks pass, but this optimization is
-not yet validated on either backend.
-
-Ascend NPU does not use this selection path. `NPUARModelRunner.sample_tokens()`
-fully constructs `OmniModelRunnerOutput`, calls
-`get_omni_connector_output()`, and only then creates the asynchronous wrapper,
-so Omni payload materialization remains synchronous.
-
-!!! note
-    `use_async_omni_output`,
-    `eager_omni_postprocess_before_async_output`, and
-    `omni_pooler_payload_include_hidden` are model implementation contracts,
-    not user-facing configuration fields.
-
-## Related Files
-
-- `vllm_omni/worker/gpu_ar_model_runner.py`: Async output object, safe tensor
-  snapshots, compatibility guards, and deferred Omni output builder.
-- `vllm_omni/platforms/npu/worker/npu_ar_model_runner.py`: Separate Ascend NPU
-  runner, where Omni output materialization remains synchronous.
-- `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`: Thinker and
-  Talker opt-in behavior.
-- `vllm_omni/deploy/qwen3_omni_moe.yaml`: Default Qwen3-Omni deployment
-  settings.
-- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py`: Qwen3-TTS
-  Talker opt-in and eager postprocess behavior.
-- `vllm_omni/deploy/qwen3_tts.yaml`: Default Qwen3-TTS deployment settings.
-- `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
-  ordering, and background error-propagation tests.
-- [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
-- [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
-  Optimization context and controlled performance results.
-- [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature
-  implementation, validation, and review history.

docs/design/feature/quantization.md

@@ -1,249 +0,0 @@
-# Quantization
-
-This document describes the quantization architecture in vLLM-Omni and the
-extension points for contributors. For user-facing configuration, supported
-hardware, and method-specific instructions, see the
-[quantization user guide](../../user_guide/quantization/overview.md).
-
-## Goals and scope
-
-The design has four goals:
-
-1. Provide one `quantization_config` entry point for diffusion-only,
-   multi-stage omni/TTS, and multi-stage diffusion models.
-2. Reuse vLLM's quantization registry and `QuantizationConfig` contract where
-   a vLLM backend already provides the required behavior.
-3. Allow different components of one pipeline to use different quantization
-   configurations.
-4. Preserve the distinction between online quantization of a BF16/FP16
-   checkpoint and loading a pre-quantized checkpoint with serialized scales.
-
-This document covers configuration resolution, component routing, model
-integration, backend extension, and validation. Kernel implementations and
-user-facing support matrices are outside its scope.
-
-## Architecture
-
-The quantization path is split across the following layers:
-
-| Layer | Responsibility | Source |
-| --- | --- | --- |
-| Public API | Exposes `build_quant_config`, component routing, and backend registration | `vllm_omni/quantization/__init__.py` |
-| Factory | Normalizes method names, selects Omni overrides or vLLM registry backends, detects ModelOpt configs, and reconciles checkpoint metadata | `vllm_omni/quantization/factory.py` |
-| Component router | Resolves a quantization config from a layer prefix using longest-prefix matching | `vllm_omni/quantization/component_config.py` |
-| Method configs | Implement `QuantizationConfig` and select the platform-specific linear method | `vllm_omni/quantization/*_config.py` |
-| Diffusion configuration | Stores the canonical runtime config and propagates checkpoint configuration | `vllm_omni/diffusion/data.py` |
-| Worker and model integration | Passes the resolved config into vLLM's model configuration and model layers | `vllm_omni/diffusion/worker/diffusion_worker.py` and `vllm_omni/diffusion/models/` |
-
-The runtime contract remains vLLM's `QuantizationConfig`:
-
-```text
-user/CLI or model config
-          |
-          v
-OmniDiffusionConfig.quantization_config
-          |
-          +--> build_quant_config()
-          |        |
-          |        +--> Omni override
-          |        +--> vLLM registry backend
-          |        +--> ComponentQuantizationConfig
-          |
-          +--> checkpoint metadata reconciliation
-          |
-          +--> diffusion worker / vLLM config
-          |
-          +--> model layers call get_quant_method(layer, prefix)
-```
-
-The model layer receives either a method implementation or `None`. Returning
-`None` leaves that layer unquantized.
-
-## Configuration resolution
-
-`build_quant_config()` accepts the following forms:
-
-| Input | Resolution |
-| --- | --- |
-| `None` or `"none"` | Disable quantization. |
-| Method string, such as `"fp8"` | Resolve an Omni override first, then the vLLM quantization registry. |
-| Flat dictionary with `method` or `quant_method` | Normalize the method and construct its `QuantizationConfig`. |
-| Per-component dictionary | Construct a `ComponentQuantizationConfig`. |
-| Existing `QuantizationConfig` | Pass through without rebuilding it. |
-
-Method aliases are normalized case-insensitively, with `-` and `_` treated as
-equivalent. `auto-round` and `auto_round` use the Omni INC/AutoRound adapter.
-ModelOpt checkpoint metadata is detected from its method, producer metadata, or
-`quant_algo`, then mapped to the corresponding vLLM ModelOpt configuration.
-
-The factory uses Omni-specific overrides for methods whose diffusion behavior
-or platform dispatch is not provided by the vLLM registry. The current override
-set includes INT8, BitsAndBytes, MXFP8, MXFP4, MXFP4 dual-scale, and INC/
-AutoRound. Other methods, such as FP8, GGUF, and ModelOpt, are resolved through
-vLLM's registry when their configuration is compatible.
-
-## Per-component routing
-
-Multi-stage models do not necessarily want one quantization policy for every
-module. A component map expresses that policy without requiring each pipeline
-to construct separate vLLM configurations:
-
-```python
-quantization_config = {
-    "transformer": {"method": "fp8"},
-    "vae": None,
-    "default": None,
-}
-```
-
-`ComponentQuantizationConfig` applies the following rules:
-
-1. Component keys are interpreted as runtime layer-prefixes.
-2. The longest matching prefix wins.
-3. `None` disables quantization for the matching component.
-4. `default`, when present, handles prefixes with no explicit match.
-5. A matching child config delegates to its normal
-   `get_quant_method(layer, prefix)` implementation.
-
-Prefix matching happens after the model's weight-name mapping. New model
-integrations must verify the actual runtime prefixes, especially when a
-`WeightsMapper` changes checkpoint names. A prefix mismatch can silently route
-a layer to the default configuration, so component-routing tests should cover
-both positive matches and fall-through behavior.
-
-## Online and pre-quantized checkpoints
-
-The same method name can represent two different loading paths:
-
-- **Online**: the source checkpoint contains BF16/FP16 weights; the method
-  creates quantized weights or scales while the model is loaded.
-- **Pre-quantized**: the checkpoint contains quantized weights and the metadata
-  required by the method; the loader consumes those serialized tensors.
-
-Method configuration owns the mode flag, for example
-`is_checkpoint_int8_serialized` or `is_checkpoint_mxfp8_serialized`. The
-configuration's `get_quant_method()` selects the corresponding linear method.
-
-For pipelines with transformer-specific `config.json` files, such as cascade
-models, `resolve_quant_config_from_disk()` reconciles the active configuration
-with each transformer's checkpoint metadata. Its invariants are:
-
-- If no active configuration exists, valid checkpoint metadata can be used for
-  auto-detection.
-- A method mismatch is an error rather than an implicit conversion.
-- Serialized-checkpoint flags rebuild an online configuration into the
-  matching offline configuration.
-- Checkpoint-specific `ignored_layers` are preserved by rebuilding the active
-  configuration when they differ.
-- AutoRound MXFP8 metadata (`data_type="mx_fp"`) selects its offline path.
-
-This prevents a model from interpreting serialized weights with an online
-loader or silently applying the wrong method to one transformer in a
-multi-transformer pipeline.
-
-## Model and layer integration
-
-Diffusion model constructors receive the resolved configuration and pass it to
-quantizable vLLM layers. Each method configuration is responsible for:
-
-- identifying supported layers in `get_quant_method()`;
-- creating the correct parameter shapes and loaders;
-- processing weights after loading when conversion is required; and
-- applying the platform-specific quantized operation at runtime.
-
-Model code must keep precision-sensitive or unsupported modules unquantized.
-The shared `safe_quant_config()` helper is used by model integrations for
-normalization and modulation layers where a generic FP8 configuration would be
-unsafe. Text encoders, audio/vision encoders, VAEs, schedulers, and tokenizers
-are not implicitly quantized just because the main transformer is quantized;
-they require explicit component routing and method support.
-
-Weight quantization and quantized KV cache are separate paths. The former is
-resolved through `quantization_config`; runtime attention/KV-cache quantization
-is configured by the diffusion attention configuration and must be validated
-independently.
-
-## Platform and parallelism boundaries
-
-Platform checks belong in the method implementation rather than in the common
-factory. This keeps configuration parsing portable while allowing CUDA, NPU,
-and XPU methods to select different kernels or reject unsupported execution at
-model construction time. The user guide is the source of truth for the current
-hardware support matrix.
-
-Quantization methods must also respect the weight layout produced by the
-configured parallelism strategy:
-
-- Tensor-parallel layers may receive partitioned weight shapes and loaders.
-- Sequence-parallel execution changes activation movement but does not by
-  itself change the quantization configuration.
-- Data-parallel replicas build their own method state; quantization metadata
-  must be available consistently on every rank.
-- `WeightsMapper` transformations must be applied before matching ignored-layer
-  lists or component prefixes.
-
-Distributed layerwise offload has an additional boundary: its sharded mmap
-AllGather path is not compatible with online quantization because the path
-expects already materialized checkpoint weights. Use the rank-local
-`--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when
-combining these features. See the [DLO design document](distributed_layerwise_offload.md)
-for the full compatibility matrix.
-
-## Adding a quantization backend
-
-Contributors should follow this sequence:
-
-1. Check whether an existing vLLM backend can provide the required weight
-   format and kernel behavior. Prefer reuse over a duplicate Omni backend.
-2. If Omni-specific behavior is required, implement a `QuantizationConfig` and
-   its `QuantizeMethodBase`/linear method in `vllm_omni/quantization/`.
-3. Implement `get_name()`, supported activation dtypes, minimum capability,
-   checkpoint parsing, ignored-layer handling, and `get_quant_method()`.
-4. Add a lazy factory override in `factory.py`. Keep optional platform
-   dependencies out of package import paths and normalize aliases to one
-   canonical method name.
-5. If the method is pre-quantized, define the checkpoint metadata and the
-   serialized loading path before adding an online path.
-6. Add model-specific component routing only when the model contains stages
-   that must use different policies.
-7. Document user-visible configuration in the quantization user guide and keep
-   implementation rationale here.
-
-The public factory registration hook,
-`register_quantization_override(method, builder)`, is intended for integrations
-that need to register an Omni-compatible builder without changing the vLLM
-registry.
-
-## Validation
-
-Validation should cover the complete path, not only whether a config object can
-be instantiated:
-
-| Area | Required checks |
-| --- | --- |
-| Factory | Strings, aliases, flat dictionaries, invalid inputs, passthrough, and component maps |
-| Method config | Online/offline selection, ignored layers, platform guards, and weight loading |
-| Checkpoint metadata | Auto-detection, method mismatch errors, serialized flags, and per-transformer metadata |
-| Model integration | Correct quantization scope, unquantized components, and parallel weight layouts |
-| Numerical quality | Same-seed BF16 versus quantized output comparison at representative resolutions and step counts |
-| Performance | Peak device memory, generation latency, and any backend-specific throughput metric |
-
-The unit tests live under `tests/diffusion/quantization/`. The trajectory
-similarity tool in
-`vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py`
-provides a repeatable quality and memory comparison for diffusion models.
-Method support and feature combinations remain model- and hardware-dependent;
-passing factory tests alone does not establish production support.
-
-## Design invariants
-
-- `quantization_config` is the single runtime source of truth after config
-  normalization.
-- A checkpoint method mismatch fails loudly instead of risking weight
-  corruption.
-- Components without an explicit supported quantization path remain in their
-  original precision.
-- Optional backend dependencies are imported lazily.
-- Method implementations own platform-specific capability checks and kernels.
-- User-facing support claims require model, hardware, numerical, and memory
-  validation in addition to configuration-level tests.

docs/design/feature/ray_based_execution.md

@@ -0,0 +1,59 @@
+# Distributed utils
+
+This directory (vllm_omni/distributed/ray_utils) contains utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.
+## 1. Installation
+```bash
+pip install "ray[default]"
+```
+## 2. Ray Utils
+
+The `ray_utils` module provides helper functions for managing Ray clusters and actors, which is essential for:
+*   **Multi-node deployment**: Running pipeline stages across different physical machines.
+*   **Resource management**: Efficient GPU/CPU allocation.
+
+### 2.1 Basic Usage
+
+To use the Ray backend, specify `worker_backend="ray"` when initializing the engine.
+
+**Command Line Example:**
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B \
+  --omni \
+  --port 8091 \
+  --worker-backend ray \
+  --ray-address auto
+```
+
+### 2.2 Cluster Setup
+
+**Step 1: Start Head Node**
+Run this on your primary machine:
+```bash
+ray start --head --port=6399
+```
+
+**Step 2: Connect Worker Nodes**
+Run this on each worker machine:
+```bash
+ray start --address=<HEAD_NODE_IP>:6399
+```
+
+> **Tip**: For a complete cluster setup script, refer to the vLLM example:
+> [run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh)
+
+### 2.3 Distributed Connector Support
+
+When running on Ray, the system automatically adapts its communication strategy:
+
+*   **Cross-Node**: Recommended to use `MooncakeTransferEngineConnector` (RDMA, fastest) or `MooncakeStoreConnector` (TCP fallback).
+*   **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma).
+*   **SHM payload routing**: `SharedMemoryConnector` always stores payloads in `/dev/shm`.
+
+### 2.4 Internal Helpers
+
+*   **`initialize_ray_cluster`**: Connects to an existing Ray cluster or starts a local one.
+
+## 3. Troubleshooting
+
+*   **Connection Issues**: Ensure the Ray head node is accessible and ports (default 6399 in this example) are open.
+*   **Version Mismatch**: Ensure all nodes run the same version of Ray and Python.

docs/design/feature/tensor_parallel.md

@@ -264,7 +264,7 @@ Complete examples in the codebase:
 | **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Standard TP | Full implementation with validation |
 | **FLUX** | `vllm_omni/diffusion/models/flux/flux_transformer.py` | Dual-stream | Image + text streams |
 | **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Standard TP | With RoPE |
-| **TP Tests** | `tests/diffusion/distributed/test_tensor_parallel.py` | E2E testing | TP correctness and performance |
+| **TP Tests** | `tests/e2e/offline_inference/test_zimage_parallelism.py` | E2E testing | TP correctness and performance |
 | **Constraint Tests** | `tests/diffusion/models/z_image/test_zimage_tp_constraints.py` | Unit testing | Validation logic |
 
 ---

docs/design/index.md

@@ -1,9 +1,6 @@
 # Design Documents
 
-This section contains design documents and architecture specifications for
+This section contains design documents and architecture specifications for vLLM-Omni.
-vLLM-Omni. The sidebar groups documents by the system concern they describe;
-retired module pages are preserved in the legacy archive instead of remaining
-in the active navigation.
 
 ## Architecture Documents
 
@@ -11,80 +8,18 @@ in the active navigation.
 
 ## Feature Design Documents
 
-### Runtime and stage execution
-
 - [Disaggregated Inference](feature/disaggregated_inference.md)
-- [Async Chunk](feature/async_chunk.md)
+- [Ray-based Execution](feature/ray_based_execution.md)
-- [Async Diffusion Output](feature/async_diffusion_output.md)
+- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
-- [Async Omni Output Materialization](feature/omni_async_output_materialization.md)
+- [Request-Level Batching for Diffusion](feature/diffusion_request_level_batching.md)
-- [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
+- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
 
-### Communication
+## Infrastructure Design Documents
-
-#### OmniConnector implementations
-
-- [Mooncake Store Connector](feature/omni_connectors/mooncake_store_connector.md)
-- [Mooncake Transfer Engine Connector](feature/omni_connectors/mooncake_transfer_engine_connector.md)
-- [Mori Transfer Engine Connector](feature/omni_connectors/mori_transfer_engine_connector.md)
-- [Shared Memory Connector](feature/omni_connectors/shared_memory_connector.md)
-- [Yuanrong Store Connector](feature/omni_connectors/yuanrong_connector.md)
-- [Yuanrong Transfer Engine Connector](feature/omni_connectors/yuanrong_transfer_engine_connector.md)
-
-### Diffusion acceleration
-
-#### Parallelism
-
-- [CFG-Parallel](feature/cfg_parallel.md)
-- [Expert Parallel](feature/expert_parallel.md)
-- [Hybrid Sharded Data Parallel (HSDP)](feature/hsdp.md)
-- [Pipeline Parallel](feature/pipeline_parallel.md)
-- [Sequence Parallel](feature/sequence_parallel.md)
-- [Tensor Parallel](feature/tensor_parallel.md)
-- [VAE Patch Parallelism](feature/vae_parallel.md)
-
-#### Attention Backends
-
-- [Skip-Softmax](feature/skip_softmax.md)
-
-#### Quantization
-
-- [Quantization](feature/quantization.md)
-
-- [Cache-DiT](feature/cache_dit.md)
-- [TeaCache](feature/teacache.md)
-- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)
-- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)
-
-## Infrastructure and Performance
 
 - [Prometheus Metrics](metrics.md)
-- [Speech Generation Performance Optimizations](qwen3_omni_tts_performance_optimization.md)
 
 ## Module Design Documents
 
-- [Entrypoints and Serving Boundaries](module/entrypoints.md)
+- [AR Module](module/ar_module.md)
-- [vLLM-Omni Configuration](module/vllm_omni_config.md)
+- [DIT Module](module/dit_module.md)
-- [Input, Output, and Modality Contracts](module/input_output_modality_contracts.md)
+- [Entrypoint Module](module/entrypoint_module.md)
-- [Error Classification, Propagation, and Rendering](module/error_contracts.md)
-- [Engine Orchestration](module/engine_orchestration.md)
-- [Stage Runtime and Replica Lifecycle](module/stage_runtime.md)
-- [OmniConnector](module/omni_connector.md)
-- [Model Integration](module/model_integration.md)
-- [Autoregressive Runtime](module/ar_runtime.md)
-- Diffusion
-  - [Overview](module/diffusion/index.md)
-  - [Runtime](module/diffusion/diffusion_runtime.md)
-  - [Model Integration](module/diffusion/diffusion_model_integration.md)
-  - [Continuous Batching](module/diffusion/continuous_batching.md)
-  - [Parallelism](module/diffusion/parallelism.md)
-  - [Offloader](module/diffusion/offloader.md)
-- [Execution Platforms](module/execution_platforms.md)
-- [Cache Management](module/cache_management.md)
-- [Quantization](module/quantization.md)
-- [Observability](module/observability.md)
-- [Profiling](module/profiling.md)
-- [Benchmarking](module/benchmarking.md)
-
-The pre-#5137 pages are preserved in the
-[legacy module archive](module/archive/README.md) for historical reference and
-are not active design contracts.

docs/design/metrics.md

@@ -148,13 +148,7 @@ Upstream vLLM's `Scheduler.make_stats()` runs on every AR generation step, retur
 
 `OmniSchedulerMixin.make_stats()` (in `vllm_omni/core/sched/omni_scheduler_mixin.py`) throttles stats emission to at most once per second **per scheduler** — i.e. per `(stage, replica)` since each replica owns its own scheduler instance. Between intervals it returns `None`, which the engine core skips serializing. This keeps gauges fresh enough for Prometheus scrapes (typically 15-30s intervals) while eliminating the per-step overhead.
 
-The orchestrator side does not add its own throttle on top. Scheduler gauges are
+The orchestrator side does not add its own throttle on top: the per-replica recording loop gates only on `raw_outputs.scheduler_stats is not None` (i.e. this replica's scheduler passed its own 1 Hz gate). A previous global `_last_stats_ts` on the orchestrator was removed because it starved every replica other than the first to emit within each second.
-recorded when `raw_outputs.scheduler_stats is not None` (i.e. this replica's
-scheduler passed its own 1 Hz gate), while output-batch request/token metrics
-are recorded from `IterationStats` whenever a non-empty output batch is
-processed. A previous global `_last_stats_ts` on the orchestrator was removed
-because it starved every replica other than the first to emit within each
-second.
 
 ## Metric Definitions
 

docs/design/module/ar_module.md

@@ -12,7 +12,7 @@ The AR module of vLLM-Omni extends vLLM's core components to support:
 - **Hidden state exposure**: Exposing per-request hidden representations for downstream stages
 - **Basic generator support**: Support some basic heterogeneous architecture such as Convolution, LSTM, etc.
 
-As shown in the [end2end example](../../../user_guide/examples/offline_inference/qwen3_omni.md), AR module can be widely applied across multiple stages, generating text tokens in thinker(AR), audio latent tokens in talker(AR) and audio wave in code2wav(Convolution).
+As shown in the [end2end example](../../user_guide/examples/offline_inference/qwen3_omni.md), AR module can be widely applied across multiple stages, generating text tokens in thinker(AR), audio latent tokens in talker(AR) and audio wave in code2wav(Convolution).
 
 ## 2. Relationship with vLLM
 

docs/design/module/ar_runtime.md

@@ -1,52 +0,0 @@
----
-title: Autoregressive Runtime
-kind: module
-status: draft
-owners:
-  - "@tzhouam"
-  - "@yinpeiqi"
-  - "@fake0fan"
-  - "@Sy0307"
-primary_code_paths:
-  - vllm_omni/core/**
-  - vllm_omni/worker/**
-related_code_paths:
-  - vllm_omni/model_executor/**
-depends_on:
-  - engine_orchestration.md
-  - model_integration.md
-  - input_output_modality_contracts.md
-validation_paths:
-  - tests/core/**
-  - tests/worker/**
-upstream_refs:
-  - vllm.v1.core
-  - vllm.v1.worker
-last_reviewed: 2026-07-16
----
-
-# Autoregressive runtime
-
-The AR runtime extends vLLM scheduling and worker execution for omni-stage
-inputs and outputs while preserving vLLM scheduling and cache semantics.
-
-## Candidate invariants
-
-### AR-INV-001: vLLM owns base scheduling semantics
-
-**Rule:** Omni schedulers MUST preserve upstream request-state and cache
-transitions unless an Omni-specific difference is documented and tested.
-
-### AR-INV-002: Omni data crosses explicit adapters
-
-**Rule:** Modality-specific stage data MUST be converted at an input or output
-adapter, not injected through unrelated scheduler state.
-
-### AR-INV-003: Workers execute assigned work
-
-**Rule:** Workers and model runners MUST NOT implement cross-stage routing.
-
-## Safe-change guide
-
-Test request lifecycle, abort, cache state, and every affected worker execution
-mode against the supported upstream vLLM contract.

docs/design/module/archive/README.md

@@ -1,18 +0,0 @@
-# Archived module design pages
-
-This directory preserves module-design pages that predate the structure
-proposed in [issue #5137](https://github.com/vllm-project/vllm-omni/issues/5137).
-They are historical snapshots, not active architectural contracts, and are
-intentionally excluded from the documentation navigation.
-
-| Archived page | Active destination |
-| --- | --- |
-| [AR Module](ar_module.md) | [Autoregressive Runtime](../ar_runtime.md) |
-| [AsyncOmni Architecture](async_omni_architecture.md) | [Engine Orchestration](../engine_orchestration.md) and [Stage Runtime](../stage_runtime.md) |
-| [DiT Module](dit_module.md) | [Diffusion module overview](../diffusion/index.md) |
-| [Entrypoint Module](entrypoint_module.md) | [Entrypoints and Serving Boundaries](../entrypoints.md) |
-
-Do not update these pages with current design guidance. Move still-useful
-material into the active destination and review it against current code and
-tests instead. No compatibility redirect pages are provided at the former
-paths.

docs/design/module/benchmarking.md

@@ -1,47 +0,0 @@
----
-title: Benchmarking
-kind: module
-status: draft
-owners:
-  - "@alex-jw-brooks"
-primary_code_paths:
-  - vllm_omni/benchmarks/**
-  - vllm_omni/entrypoints/cli/benchmark/**
-related_code_paths:
-  - benchmarks/**
-depends_on:
-  - entrypoints.md
-  - profiling.md
-validation_paths:
-  - tests/benchmarks/**
-  - tests/dfx/perf/**
-upstream_refs:
-  - vllm.benchmarks
-last_reviewed: 2026-07-16
----
-
-# Benchmarking
-
-Benchmarking defines reproducible workloads, metrics, warm-up, measurement,
-and result metadata for comparing vLLM-Omni performance.
-
-## Candidate invariants
-
-### BENCH-INV-001: Results are reproducible
-
-**Rule:** Results MUST identify the model, revision, commit, hardware,
-configuration, workload, concurrency, warm-up, and measurement window.
-
-### BENCH-INV-002: Correctness precedes performance
-
-**Rule:** A benchmark MUST validate successful and semantically acceptable
-outputs before using timings as performance evidence.
-
-### BENCH-INV-003: Metric definitions are explicit
-
-**Rule:** Metrics MUST state their unit, population, and aggregation.
-
-## Safe-change guide
-
-Test workload parsing and metric calculations with deterministic fixtures. Do
-not compare materially different workloads or runtime settings.

docs/design/module/cache_management.md

@@ -1,47 +0,0 @@
----
-title: Cache Management
-kind: module
-status: draft
-owners:
-  - "@Isotr0py"
-  - "@princepride"
-primary_code_paths:
-  - vllm_omni/diffusion/cache/**
-related_code_paths:
-  - vllm_omni/experimental/ar_diffusion/kv_cache/**
-  - vllm_omni/core/**
-depends_on:
-  - ar_runtime.md
-  - diffusion/diffusion_runtime.md
-validation_paths:
-  - tests/diffusion/cache/**
-upstream_refs:
-  - vllm.v1.core.kv_cache_manager
-last_reviewed: 2026-07-16
----
-
-# Cache management
-
-Cache management defines reusable state, cache identity, validity, lifecycle,
-and eviction across AR and diffusion execution.
-
-## Candidate invariants
-
-### CACHE-INV-001: Cache identity is complete
-
-**Rule:** A cache key MUST include every input and execution property that can
-change the reused value.
-
-### CACHE-INV-002: Reuse preserves correctness
-
-**Rule:** A cache optimization MUST provide a disabled path and MUST NOT reuse
-state after its validity conditions change.
-
-### CACHE-INV-003: Ownership has an end
-
-**Rule:** Request-scoped and model-scoped cache entries MUST have explicit
-reset, eviction, or teardown behavior.
-
-## Safe-change guide
-
-Test hit, miss, invalidation, disabled, concurrent-request, and teardown paths.

docs/design/module/diffusion/continuous_batching.md

@@ -1,51 +0,0 @@
----
-title: Diffusion Continuous Batching
-kind: module
-status: draft
-owners:
-  - "@Isotr0py"
-  - "@princepride"
-primary_code_paths:
-  - vllm_omni/diffusion/sched/**
-related_code_paths:
-  - vllm_omni/diffusion/executor/**
-depends_on:
-  - diffusion_runtime.md
-validation_paths:
-  - tests/diffusion/batching/**
-upstream_refs: []
-last_reviewed: 2026-07-16
----
-
-# Diffusion continuous batching
-
-Continuous batching defines when diffusion requests are compatible, how they
-share an execution step, and how each request advances independently.
-
-## Candidate invariants
-
-### BATCH-INV-001: Compatibility is explicit
-
-**Rule:** Requests MAY share a batch only when all properties required by the
-selected pipeline, scheduler step, shape, precision, and execution features are
-compatible.
-
-### BATCH-INV-002: Per-request state remains isolated
-
-**Rule:** Batch assembly MUST preserve each request's random generator, progress,
-conditioning, outputs, cancellation, and errors.
-
-### BATCH-INV-003: Admission is bounded
-
-**Rule:** Admission MUST respect configured capacity and MUST NOT rely on an
-unbounded waiting or active-request collection.
-
-### BATCH-INV-004: Batch order is not completion order
-
-**Rule:** Output association MUST use stable request identity rather than batch
-position across execution steps.
-
-## Safe-change guide
-
-Test heterogeneous requests, partial completion, cancellation, deterministic
-seeds, capacity limits, and batch sizes of one and many.

docs/design/module/diffusion/diffusion_model_integration.md

@@ -1,58 +0,0 @@
----
-title: Diffusion Model Integration
-kind: module
-status: draft
-owners:
-  - "@Isotr0py"
-  - "@wtomin"
-  - "@ZJY0516"
-primary_code_paths:
-  - vllm_omni/diffusion/models/**
-  - vllm_omni/diffusion/model_loader/**
-related_code_paths:
-  - vllm_omni/diffusion/layers/**
-  - vllm_omni/diffusion/lora/**
-  - vllm_omni/diffusion/utils/**
-depends_on:
-  - diffusion_runtime.md
-  - ../input_output_modality_contracts.md
-validation_paths:
-  - tests/diffusion/models/**
-  - tests/diffusion/model_loader/**
-  - tests/diffusion/layers/**
-  - tests/diffusion/lora/**
-upstream_refs:
-  - diffusers.DiffusionPipeline
-last_reviewed: 2026-07-16
----
-
-# Diffusion model integration
-
-Diffusion model integration owns pipeline contracts, registration, checkpoint
-loading, adapters, shared layers, and model-specific processing.
-
-## Candidate invariants
-
-### DIFF-MODEL-INV-001: Pipelines implement one runtime contract
-
-**Rule:** A pipeline MUST declare its supported modalities, configuration,
-inputs, outputs, loading path, and runtime capabilities.
-
-### DIFF-MODEL-INV-002: Registration is the selection boundary
-
-**Rule:** Runtime code MUST select model implementations through the registry or
-loader contract rather than scattered model-name conditionals.
-
-### DIFF-MODEL-INV-003: Model code does not schedule requests
-
-**Rule:** Pipeline code MUST NOT own admission, batching, cancellation, or
-cross-stage routing.
-
-### DIFF-MODEL-INV-004: Shared behavior stays shared
-
-**Rule:** Model directories SHOULD contain only genuine model differences.
-
-## Safe-change guide
-
-Test registry selection, checkpoint loading, minimal inference, input and output
-contracts, and every declared optional capability.

docs/design/module/diffusion/diffusion_runtime.md

@@ -1,58 +0,0 @@
----
-title: Diffusion Runtime
-kind: module
-status: draft
-owners:
-  - "@Isotr0py"
-  - "@princepride"
-  - "@SamitHuang"
-primary_code_paths:
-  - vllm_omni/diffusion/sched/**
-  - vllm_omni/diffusion/executor/**
-  - vllm_omni/diffusion/worker/**
-related_code_paths:
-  - vllm_omni/diffusion/hooks/**
-  - vllm_omni/diffusion/postprocess/**
-depends_on:
-  - ../engine_orchestration.md
-  - ../input_output_modality_contracts.md
-  - diffusion_model_integration.md
-validation_paths:
-  - tests/diffusion/diffusion_backend/**
-  - tests/diffusion/hooks/**
-upstream_refs:
-  - diffusers.DiffusionPipeline
-last_reviewed: 2026-07-16
----
-
-# Diffusion runtime
-
-The diffusion runtime owns request admission, scheduling, execution, progress,
-output, cancellation, and cleanup inside a diffusion stage.
-
-## Candidate invariants
-
-### DIFF-RUNTIME-INV-001: One lifecycle owner
-
-**Rule:** Every admitted request MUST have exactly one scheduler-owned lifecycle
-until completion, cancellation, or failure.
-
-### DIFF-RUNTIME-INV-002: Execution follows scheduler output
-
-**Rule:** Executors and workers MUST execute scheduler decisions without
-admitting, reordering, or forwarding requests independently.
-
-### DIFF-RUNTIME-INV-003: Terminal cleanup is complete
-
-**Rule:** Every terminal path MUST release request state, temporary tensors,
-hooks, and runtime-owned resources.
-
-### DIFF-RUNTIME-INV-004: Optional features use runtime hooks
-
-**Rule:** Cache, profiling, offload, and parallel features SHOULD integrate at
-defined hooks instead of creating another request lifecycle.
-
-## Safe-change guide
-
-Test success, cancellation, failure, shutdown, multiple requests, and resource
-cleanup.

docs/design/module/diffusion/index.md

@@ -1,42 +0,0 @@
----
-title: Diffusion Modules
-kind: module-index
-status: draft
-owners:
-  - "@Isotr0py"
-  - "@princepride"
-  - "@SamitHuang"
-  - "@wtomin"
-  - "@ZJY0516"
-  - "@RuixiangMa"
-  - "@david6666666"
-  - "@xuechendi"
-primary_code_paths:
-  - vllm_omni/diffusion/**
-related_code_paths:
-  - vllm_omni/platforms/**
-depends_on:
-  - ../engine_orchestration.md
-  - ../input_output_modality_contracts.md
-validation_paths:
-  - tests/diffusion/**
-upstream_refs:
-  - diffusers
-last_reviewed: 2026-07-16
----
-
-# Diffusion modules
-
-Diffusion is a module family containing runtime, model integration, batching,
-distributed execution, and memory-management subsystems.
-
-## Module documents
-
-- [Diffusion runtime](diffusion_runtime.md)
-- [Diffusion model integration](diffusion_model_integration.md)
-- [Continuous batching](continuous_batching.md)
-- [Parallelism](parallelism.md)
-- [Offloader](offloader.md)
-
-Cache, quantization, profiling, and benchmarking follow their top-level module
-documents.

docs/design/module/diffusion/offloader.md

@@ -1,53 +0,0 @@
----
-title: Diffusion Offloader
-kind: module
-status: draft
-owners:
-  - "@yuanheng-zhao"
-  - "@Isotr0py"
-primary_code_paths:
-  - vllm_omni/diffusion/offloader/**
-related_code_paths:
-  - vllm_omni/diffusion/models/**
-  - vllm_omni/diffusion/worker/**
-depends_on:
-  - diffusion_runtime.md
-  - ../execution_platforms.md
-validation_paths:
-  - tests/diffusion/offloader/**
-upstream_refs:
-  - torch.nn.Module.to
-last_reviewed: 2026-07-16
----
-
-# Diffusion offloader
-
-The offloader owns component residency, transfer scheduling, memory accounting,
-prefetch, and teardown for diffusion execution.
-
-## Candidate invariants
-
-### OFFLOAD-INV-001: Residency is explicit
-
-**Rule:** Each managed component MUST have one known residency state and one
-owner responsible for transitions.
-
-### OFFLOAD-INV-002: Use follows readiness
-
-**Rule:** Runtime execution MUST NOT consume a component until its transfer to
-the required device is complete.
-
-### OFFLOAD-INV-003: Transfers preserve model state
-
-**Rule:** Offloading MUST preserve parameter and buffer identity, dtype, device,
-and correctness required by the selected execution mode.
-
-### OFFLOAD-INV-004: Memory is bounded and released
-
-**Rule:** Retained host and device copies MUST respect configured limits and
-provide deterministic teardown.
-
-## Safe-change guide
-
-Test enabled and disabled paths, repeated execution, asynchronous transfer,
-memory limits, failure, teardown, and numerical equivalence.

docs/design/module/diffusion/parallelism.md

@@ -1,56 +0,0 @@
----
-title: Diffusion Parallelism
-kind: module
-status: draft
-owners:
-  - "@princepride"
-  - "@yuanheng-zhao"
-  - "@xuechendi"
-primary_code_paths:
-  - vllm_omni/diffusion/distributed/**
-  - vllm_omni/diffusion/attention/parallel/**
-related_code_paths:
-  - vllm_omni/distributed/**
-  - vllm_omni/config/composable_parallel/**
-depends_on:
-  - diffusion_runtime.md
-  - ../vllm_omni_config.md
-validation_paths:
-  - tests/diffusion/distributed/**
-  - tests/diffusion/attention/**
-upstream_refs:
-  - torch.distributed
-last_reviewed: 2026-07-16
----
-
-# Diffusion parallelism
-
-Parallelism owns diffusion rank topology, process groups, tensor and sequence
-sharding, collectives, and distributed execution.
-
-## Candidate invariants
-
-### PARALLEL-INV-001: Topology has one source of truth
-
-**Rule:** Rank coordinates, group membership, and parallel dimensions MUST be
-derived from validated configuration.
-
-### PARALLEL-INV-002: Sharding contracts are explicit
-
-**Rule:** Every distributed boundary MUST define tensor shape, sharded
-dimension, placement, dtype, producer, and consumer.
-
-### PARALLEL-INV-003: Collectives are symmetric
-
-**Rule:** Process-group members MUST invoke compatible collectives in the same
-logical order.
-
-### PARALLEL-INV-004: Single-rank behavior remains valid
-
-**Rule:** Distributed implementations SHOULD preserve a supported single-rank
-path without distributed initialization.
-
-## Safe-change guide
-
-Test topology validation, single-rank execution, affected parallel modes,
-combined modes, invalid shapes, and orderly teardown.

docs/design/module/dit_module.md

@@ -252,7 +252,8 @@ while True:
   batch of compatible independent requests for one full pipeline forward when
   the pipeline supports request-level batching; the step-wise path can also
   admit or remove compatible requests between denoise steps. See
-  [Diffusion Continuous Batching](../../feature/diffusion_continuous_batching.md).
+  [Request-Level Batching](../feature/diffusion_request_level_batching.md) and
+  [Continuous Batching for Step-Wise Diffusion](../feature/diffusion_continuous_batching.md).
 
 ---
 
@@ -539,9 +540,9 @@ def get_attn_backend_for_role(role, head_size, attention_config=None, role_categ
 
 4. **Platform default** — `current_omni_platform.get_diffusion_attn_backend_cls(...)` picks the best-available kernel for the hardware.
 
-A backend that needs configuration exposes it as a typed field on `AttentionSpec`; `AttentionSpec.backend_kwargs()` serializes it into the `backend_kwargs` the impl consumes. Today those fields are `skip_softmax` (TRTLLM_ATTN) and `block_sparse` (any block-sparse backend, currently RAINFUSION_ATTN), so e.g. `target_sparsity` or `sparsity` can be set without changing model code.
+A backend that needs configuration exposes it as a typed field on `AttentionSpec`; `AttentionSpec.backend_kwargs()` serializes it into the `backend_kwargs` the impl consumes. Today the only such field is `skip_softmax` (TRTLLM_ATTN), so e.g. `target_sparsity` can be set without changing model code.
 
-For the user-facing CLI surface, see [Diffusion Attention Backends](../../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../../contributing/model/adding_diffusion_model.md).
+For the user-facing CLI surface, see [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../contributing/model/adding_diffusion_model.md).
 
 **Backend Availability**:
 

docs/design/module/engine_orchestration.md

@@ -1,122 +0,0 @@
----
-title: Engine Orchestration
-kind: module
-status: draft
-architecture_state: current-plus-in-flight
-owners:
-  - "@tzhouam"
-  - "@fake0fan"
-document_stewards:
-  - "@hsliuustc0106"
-  - "@Gaohan123"
-  - "@david6666666"
-required_reviewers:
-  - "@yinpeiqi"
-  - "@Sy0307"
-  - "@yenuo26"
-  - "@NickCao"
-primary_code_paths:
-  - vllm_omni/engine/async_omni_engine.py
-  - vllm_omni/engine/async_engine_utils.py
-  - vllm_omni/engine/orchestrator.py
-  - vllm_omni/engine/cfg_companion_tracker.py
-  - vllm_omni/engine/rpc_result_router.py
-related_code_paths:
-  - vllm_omni/engine/messages.py
-  - vllm_omni/engine/orchestrator_monitor.py
-  - vllm_omni/engine/stage_client.py
-  - vllm_omni/engine/stage_pool.py
-  - vllm_omni/outputs/output_processor.py
-  - vllm_omni/distributed/omni_coordinator/**
-depends_on:
-  - input_output_modality_contracts.md
-  - error_contracts.md
-  - stage_runtime.md
-  - omni_connector.md
-validation_paths:
-  - tests/engine/test_async_omni_engine_input.py
-  - tests/engine/test_async_omni_engine_outputs.py
-  - tests/engine/test_async_omni_engine_abort.py
-  - tests/engine/test_async_omni_engine_stage_init.py
-  - tests/engine/test_orchestrator.py
-  - tests/engine/test_orchestrator_error_handling.py
-  - tests/engine/test_orchestrator_stage_input_bridge.py
-  - tests/engine/test_cfg_companion_lifecycle.py
-  - tests/engine/test_rpc_result_router.py
-  - tests/e2e/features/fullduplex/engine/**
-upstream_refs:
-  - vllm.v1.engine.EngineCoreRequest
-  - vllm.v1.engine.EngineCoreOutputs
-  - vllm.v1.engine.exceptions.EngineDeadError
-  - vllm.outputs.RequestOutput
-invariant_namespace: ORCH-INV
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# Engine orchestration
-
-Engine orchestration coordinates configured AR and diffusion stages while
-keeping public clients independent from stage processes and transports.
-
-## Contract status
-
-This document is a draft description of current behavior. It also identifies
-the boundary affected by the in-flight stage client/process refactor in
-[#5441](https://github.com/vllm-project/vllm-omni/pull/5441). Names and
-responsibilities proposed only by that PR are not current contracts.
-
-## Ownership boundary
-
-This document owns `AsyncOmniEngine`, `Orchestrator`, request-state creation,
-cross-stage routing, output ordering, companion tracking, control/RPC
-correlation, cancellation propagation, and terminal-state convergence.
-
-It does not own stage placement, replica selection, stage-process startup,
-payload schemas, public protocol rendering, connector transport, or semantic
-error classification. Those responsibilities belong to the stage runtime,
-I/O, entrypoint, connector, and error contracts.
-
-## Candidate invariants
-
-These identifiers are proposals while the document is `draft`.
-
-### ORCH-INV-001: The orchestrator owns cross-stage routing
-
-**Rule:** Entrypoints and stage clients MUST NOT independently forward a
-request to a downstream logical stage.
-
-### ORCH-INV-002: Stage clients do not own routing policy
-
-**Rule:** Stage clients MUST implement communication and lifecycle operations
-without selecting the next logical stage.
-
-### ORCH-INV-100: Terminal state is monotonic
-
-**Rule:** Once a request reaches a terminal state, orchestration MUST NOT
-forward new work for that request.
-
-## Invariant namespace
-
-`ORCH-INV` reserves `001-099` for dependency direction, `100-199` for
-request state and ordering, `200-299` for failure/cancellation/cleanup, and
-`300-399` for extension points, shutdown ordering, and upstream alignment.
-Numbers become append-only after normative promotion.
-
-## Safe-change guide
-
-Test routing, output ordering, queue correlation, cancellation, failure
-propagation, shutdown ordering, and representative multi-stage execution.
-Changes that cross into stage runtime or public error behavior require review
-from that contract's owners.
-
-## Promotion gate
-
-- Reconcile current names and boundaries after #5441 merges, closes, or is
-  replaced.
-- Demonstrate one terminal outcome per request across success, abort,
-  request-scoped failure, and fatal engine failure.
-- Verify queue correlation and orchestrator-before-runtime shutdown ordering
-  in the cited validation paths.
-- Obtain approval from a technical owner and an independent validation
-  reviewer.

docs/design/module/entrypoints.md

@@ -1,134 +0,0 @@
----
-title: Entrypoints and Serving Boundaries
-kind: module
-status: draft
-architecture_state: current-plus-in-flight
-owners:
-  - "@alex-jw-brooks"
-  - "@linyueqian"
-document_stewards:
-  - "@hsliuustc0106"
-  - "@Gaohan123"
-  - "@david6666666"
-required_reviewers:
-  - "@tzhouam"
-  - "@herotai214"
-  - "@yenuo26"
-  - "@NickCao"
-primary_code_paths:
-  - vllm_omni/entrypoints/omni.py
-  - vllm_omni/entrypoints/async_omni.py
-  - vllm_omni/entrypoints/omni_base.py
-  - vllm_omni/entrypoints/cli/**
-  - vllm_omni/entrypoints/openai/**
-  - vllm_omni/entrypoints/openpi/**
-  - vllm_omni/entrypoints/client_request_state.py
-  - vllm_omni/entrypoints/stage_utils.py
-  - vllm_omni/entrypoints/utils.py
-primary_path_exceptions:
-  - path: vllm_omni/entrypoints/openai/errors.py
-    owner: error_contracts.md
-related_code_paths:
-  - vllm_omni/errors.py
-  - vllm_omni/inputs/**
-  - vllm_omni/outputs/**
-  - vllm_omni/engine/async_omni_engine.py
-  - vllm_omni/config/**
-  - vllm_omni/deploy/**
-depends_on:
-  - input_output_modality_contracts.md
-  - error_contracts.md
-  - engine_orchestration.md
-validation_paths:
-  - tests/entrypoints/test_omni_entrypoints.py
-  - tests/entrypoints/test_async_omni.py
-  - tests/entrypoints/test_async_omni_duplex.py
-  - tests/entrypoints/test_serve.py
-  - tests/entrypoints/test_stream_finish_reason.py
-  - tests/entrypoints/openai/**
-  - tests/entrypoints/openai_api/**
-  - tests/e2e/online_serving/**
-upstream_refs:
-  - vllm.engine.protocol.EngineClient
-  - vllm.entrypoints.openai/**
-  - vllm.entrypoints.serve/**
-  - vllm.renderers/**
-  - vllm.v1.engine.exceptions/**
-invariant_namespace: ENTRY-INV
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# Entrypoints and serving boundaries
-
-Entrypoints translate offline, CLI, and serving requests into stable engine
-operations and translate engine outputs into public responses.
-
-## Contract status
-
-This document describes current entrypoint responsibilities plus the boundary
-under review in the open roadmap
-[#5227](https://github.com/vllm-project/vllm-omni/issues/5227) and helper-move
-PR [#5453](https://github.com/vllm-project/vllm-omni/pull/5453). In-flight
-helper locations are not treated as current paths.
-
-## Ownership boundary
-
-This document owns offline API semantics, CLI and serve composition,
-supported OpenAI-compatible routes, request validation and normalization,
-response conversion, streaming/session behavior, and engine handoff.
-
-It does not own configuration precedence, cross-stage routing, stage
-placement, payload implementation, or semantic error classification.
-`entrypoints/openai/errors.py` is an explicit primary-path exception owned by
-`error_contracts.md`.
-
-## Candidate invariants
-
-These identifiers are proposals while the document is `draft`.
-
-### ENTRY-INV-001: Entrypoints adapt but do not orchestrate
-
-**Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
-policy.
-
-### ENTRY-INV-100: Public requests are normalized once
-
-**Rule:** Public protocol values MUST be validated and converted to internal
-request contracts before engine submission.
-
-### ENTRY-INV-101: Streaming preserves request identity
-
-**Rule:** Every streamed response MUST remain associated with the request and
-output modality that produced it.
-
-The reviewer-proposed rule that model-specific behavior should stay behind a
-common adapter or processor abstraction remains an unnumbered candidate until
-the entrypoint refactor makes that boundary enforceable.
-
-## Invariant namespace
-
-`ENTRY-INV` reserves `001-099` for boundary and dependency direction,
-`100-199` for normalization, sessions, and streaming, `200-299` for
-disconnect, cancellation, rendering, and cleanup, and `300-399` for upstream
-route and renderer compatibility. Numbers become append-only after normative
-promotion.
-
-## Safe-change guide
-
-Test request validation, protocol conversion, model-adapter routing,
-streaming, session identity, disconnect/cancellation, and error mapping for
-each affected offline or serving entrypoint.
-
-## Promotion gate
-
-- Reconcile the page after #5227's P0 ownership work and #5453's final
-  disposition.
-- Publish a supported route/transport matrix rather than claiming blanket
-  OpenAI parity.
-- Verify one normalization and handoff path per route family and one terminal
-  outcome per streaming transport.
-- Promote the `api_server.py` composition-root rule only after helper moves and
-  import boundaries enforce it.
-- Obtain approval from a technical owner and an independent validation
-  reviewer.

docs/design/module/error_contracts.md

@@ -1,109 +0,0 @@
----
-title: Error Classification, Propagation, and Rendering
-kind: module
-status: draft
-architecture_state: current-plus-rfc
-owners:
-  - "@alex-jw-brooks"
-  - "@NickCao"
-document_stewards:
-  - "@hsliuustc0106"
-  - "@Gaohan123"
-  - "@david6666666"
-required_reviewers:
-  - "@yinpeiqi"
-  - "@tzhouam"
-  - "@yenuo26"
-primary_code_paths:
-  - vllm_omni/errors.py
-  - vllm_omni/entrypoints/openai/errors.py
-related_code_paths:
-  - vllm_omni/engine/messages.py
-  - vllm_omni/engine/orchestrator.py
-  - vllm_omni/entrypoints/omni_base.py
-  - vllm_omni/entrypoints/async_omni.py
-  - vllm_omni/entrypoints/openai/api_server.py
-  - vllm_omni/entrypoints/openai/serving_chat.py
-  - vllm_omni/entrypoints/openai/serving_speech.py
-  - vllm_omni/entrypoints/openai/serving_audio_generate.py
-  - vllm_omni/entrypoints/openai/protocol/images.py
-  - vllm_omni/entrypoints/openai/protocol/videos.py
-depends_on: []
-validation_paths:
-  - tests/engine/test_orchestrator_error_handling.py
-  - tests/engine/test_async_omni_engine_outputs.py
-  - tests/entrypoints/test_omni_entrypoints.py
-  - tests/entrypoints/test_stream_finish_reason.py
-  - tests/entrypoints/openai_api/test_image_server.py
-  - tests/entrypoints/openai_api/test_video_server.py
-  - tests/entrypoints/openai_api/test_serving_audio_generate.py
-  - tests/entrypoints/openai_api/test_serving_speech.py
-  - tests/metrics/test_prometheus.py
-upstream_refs:
-  - vllm.v1.engine.exceptions.EngineDeadError
-  - vllm.v1.engine.exceptions.EngineGenerateError
-  - vllm.entrypoints.serve.utils.error_response
-  - vLLM RFC #48227
-  - vLLM PR #49665
-invariant_namespace: ERR-INV
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# Error classification, propagation, and rendering
-
-This contract separates the meaning of a failure from the transport used to
-carry it and the public protocol used to render it.
-
-## Contract status
-
-Current behavior is distributed across engine, message, and entrypoint code.
-The semantic hierarchy proposed by
-[#5570](https://github.com/vllm-project/vllm-omni/issues/5570) remains an open
-RFC. Proposed types and wire records from that RFC are not current contracts.
-
-## Ownership boundary
-
-This document owns semantic classification, fatal versus request-scoped
-failure, cross-process error context, compatibility policy, sanitization, and
-offline, HTTP, SSE, WebSocket, and asynchronous-job rendering rules.
-
-`engine/messages.py` remains primarily owned by the I/O contract; this page
-owns the semantics of its error fields. Route and serving files remain
-primarily owned by `entrypoints.md`; this page owns their rendering
-constraints.
-
-## Candidate invariants
-
-The following topics require owner review before receiving stable IDs:
-
-- A request-scoped failure terminates only the affected request unless the
-  underlying engine is no longer usable.
-- Every public rendering path preserves a stable classification while
-  removing sensitive internal context.
-- Cancellation remains distinguishable from internal failure across process
-  and protocol boundaries.
-
-## Invariant namespace
-
-`ERR-INV` reserves `001-099` for classification ownership, `100-199` for wire
-context and compatibility, `200-299` for fatality, termination, sanitization,
-and rendering, and `300-399` for upstream alignment and transport extension
-points. Numbers become append-only after normative promotion.
-
-## Safe-change guide
-
-Exercise exception classification, stage wire round trips, fatal and
-request-scoped propagation, cancellation, sanitization, offline exceptions,
-HTTP status/body consistency, streaming terminal events, asynchronous-job
-persistence, and metrics classification.
-
-## Promotion gate
-
-- Resolve #5570's hierarchy, wire record, sanitization, and mixed-version
-  questions.
-- Add table-driven exception/status/type tests and stage wire round trips.
-- Verify all public transports terminate once and do not leak sensitive
-  payload data.
-- Obtain approval from both technical owners and an independent validation
-  reviewer.

docs/design/module/execution_platforms.md

@@ -1,48 +0,0 @@
----
-title: Execution Platforms
-kind: module
-status: draft
-owners:
-  - "@gcanlin"
-  - "@tjtanaa"
-  - "@xuechendi"
-primary_code_paths:
-  - vllm_omni/platforms/**
-related_code_paths:
-  - vllm_omni/attention/**
-depends_on:
-  - ar_runtime.md
-  - diffusion/index.md
-validation_paths:
-  - tests/platforms/**
-upstream_refs:
-  - vllm.platforms
-last_reviewed: 2026-07-16
----
-
-# Execution platforms
-
-Execution platforms isolate hardware-specific capability detection, worker
-selection, patches, kernels, and configuration adjustments.
-
-## Candidate invariants
-
-### PLATFORM-INV-001: Capabilities are explicit
-
-**Rule:** Hardware-dependent behavior MUST be guarded by platform selection or
-capability detection and MUST fail clearly when unsupported.
-
-### PLATFORM-INV-002: Portable code remains portable
-
-**Rule:** Platform-neutral modules MUST NOT import a vendor implementation
-directly when a common selection interface exists.
-
-### PLATFORM-INV-003: Overrides are minimal
-
-**Rule:** A platform implementation SHOULD override only behavior that differs
-from the common or upstream implementation.
-
-## Safe-change guide
-
-Validate common import paths without the target accelerator and run focused
-tests in a fresh `uv` environment on every affected device platform.

docs/design/module/input_output_modality_contracts.md

@@ -1,133 +0,0 @@
----
-title: Input, Output, and Modality Contracts
-kind: module
-status: draft
-architecture_state: current
-owners:
-  - "@Sy0307"
-  - "@amy-why-3459"
-document_stewards:
-  - "@hsliuustc0106"
-  - "@Gaohan123"
-  - "@david6666666"
-required_reviewers:
-  - "@tzhouam"
-  - "@linyueqian"
-  - "@david6666666"
-  - "@yenuo26"
-  - "@NickCao"
-primary_code_paths:
-  - vllm_omni/inputs/**
-  - vllm_omni/outputs/**
-  - vllm_omni/request.py
-  - vllm_omni/data_entry_keys.py
-  - vllm_omni/engine/messages.py
-  - vllm_omni/engine/serialization.py
-  - vllm_omni/engine/mm_outputs.py
-  - vllm_omni/engine/output_modality.py
-  - vllm_omni/engine/output_processor.py
-related_code_paths:
-  - vllm_omni/model_executor/stage_input_processors/**
-  - vllm_omni/core/sched/**
-  - vllm_omni/distributed/omni_connectors/**
-  - vllm_omni/diffusion/request.py
-  - vllm_omni/diffusion/worker/request_batch.py
-  - vllm_omni/errors.py
-depends_on:
-  - error_contracts.md
-validation_paths:
-  - tests/inputs/**
-  - tests/engine/test_data_entry_keys.py
-  - tests/engine/test_omni_request.py
-  - tests/engine/test_omni_request_output.py
-  - tests/engine/test_multimodal_accumulation.py
-  - tests/engine/test_output_modality.py
-  - tests/engine/test_output_processor.py
-  - tests/engine/test_wire_multimodal_output.py
-  - tests/utils/test_mm_outputs.py
-  - tests/utils/test_mm_outputs_partition.py
-  - tests/diffusion/test_diffusion_output_metadata.py
-upstream_refs:
-  - vllm.inputs/**
-  - vllm.outputs/**
-  - vllm.v1.request.Request
-  - vllm.v1.engine.EngineCoreRequest
-  - vllm.v1.engine.EngineCoreOutput
-  - vllm.v1.engine.output_processor.OutputProcessor
-invariant_namespace: IO-INV
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# Input, output, and modality contracts
-
-These contracts define the data that may cross entrypoint, orchestration,
-stage, connector, and model boundaries.
-
-## Contract status
-
-This document is a draft description of the current request, message,
-serialization, modality, accumulation, and output types. It does not assign
-semantic error ownership to the wire schema.
-
-## Ownership boundary
-
-This document owns request identity, prompt/input types, modality keys and
-metadata, queue and wire message schemas, serialization, output types,
-accumulation, completion semantics, and compatibility shims.
-
-It does not own route-specific validation or rendering, model-specific
-interpretation, connector transport mechanics, scheduling policy, or semantic
-error classification. The `ErrorMessage` schema belongs here; the meaning and
-public rendering of its error fields belong to `error_contracts.md`.
-
-## Candidate invariants
-
-These identifiers are proposals while the document is `draft`.
-
-### IO-INV-001: Boundary data has an explicit modality
-
-**Rule:** Data crossing a module or stage boundary MUST identify its modality
-and use the corresponding validated contract.
-
-### IO-INV-100: Request identity is stable
-
-**Rule:** Request identity MUST be preserved across conversions, stages,
-streaming updates, cancellation, and errors.
-
-### IO-INV-101: Completion is explicit
-
-**Rule:** Producers MUST distinguish partial updates from the terminal update
-for every output modality.
-
-### IO-INV-300: Internal objects do not leak into public protocols
-
-**Rule:** Entrypoints MUST explicitly translate internal output objects into
-public response types.
-
-## Invariant namespace
-
-`IO-INV` reserves `001-099` for schema ownership and dependency direction,
-`100-199` for identity, modality, accumulation, ordering, and completion,
-`200-299` for serialization failure and payload cleanup, and `300-399` for
-upstream extension, deprecation, and compatibility. Numbers become
-append-only after normative promotion.
-
-## Safe-change guide
-
-Test construction, validation, serialization, round trips, streaming,
-accumulation, completion, optional fields, unknown fields, and compatibility
-at every affected producer-consumer boundary.
-
-## Promotion gate
-
-- Add explicit CODEOWNERS for the primary I/O paths or confirm that fallback
-  ownership is intentional.
-- Document the compatibility window for deprecated `vllm_omni.engine`
-  I/O-related imports.
-- Verify wire round trips and rejection of unknown or incompatible schema
-  fields.
-- Define payload-versus-metadata rules and completion semantics for streaming
-  and batched outputs.
-- Obtain approval from a technical owner and independent AR, diffusion,
-  connector, entrypoint, and validation reviewers as applicable.

docs/design/module/model_integration.md

@@ -1,51 +0,0 @@
----
-title: Model Integration
-kind: module
-status: draft
-owners:
-  - "@tzhouam"
-  - "@gcanlin"
-primary_code_paths:
-  - vllm_omni/model_executor/**
-  - vllm_omni/model_extras/**
-  - vllm_omni/plugins/**
-related_code_paths:
-  - vllm_omni/transformers_utils/**
-  - vllm_omni/tokenizers/**
-depends_on:
-  - input_output_modality_contracts.md
-  - ar_runtime.md
-validation_paths:
-  - tests/model_executor/**
-  - tests/model_extras/**
-  - tests/plugin/**
-upstream_refs:
-  - vllm.model_executor
-last_reviewed: 2026-07-16
----
-
-# Model integration
-
-Model integration adapts model-specific configuration, preprocessing, loading,
-and execution behavior to stable runtime contracts.
-
-## Candidate invariants
-
-### MODEL-INV-001: Registration is explicit
-
-**Rule:** A model integration MUST declare how its model class, loader, input
-processor, and stage configuration are selected.
-
-### MODEL-INV-002: Model code does not route stages
-
-**Rule:** Model-specific code MUST NOT select or invoke a downstream omni stage.
-
-### MODEL-INV-003: Upstream behavior is reused deliberately
-
-**Rule:** An upstream vLLM implementation SHOULD be reused when its contract is
-sufficient; an override MUST document the behavioral difference.
-
-## Safe-change guide
-
-Test registration, checkpoint loading, input conversion, and representative
-model execution. Test shared utilities against more than one integration.

docs/design/module/observability.md

@@ -1,46 +0,0 @@
----
-title: Observability
-kind: module
-status: draft
-owners:
-  - "@lishunyang12"
-primary_code_paths:
-  - vllm_omni/metrics/**
-related_code_paths:
-  - vllm_omni/engine/**
-  - vllm_omni/entrypoints/**
-depends_on:
-  - engine_orchestration.md
-validation_paths:
-  - tests/metrics/**
-upstream_refs:
-  - vllm.v1.metrics
-last_reviewed: 2026-07-16
----
-
-# Observability
-
-Observability defines stable metrics, logs, and correlation fields for tracing a
-request across entrypoints, orchestration, stages, and workers.
-
-## Candidate invariants
-
-### OBS-INV-001: Request correlation survives stage boundaries
-
-**Rule:** Logs and metrics for the same request SHOULD carry a stable request
-identifier and stage identity.
-
-### OBS-INV-002: Metric meaning is stable
-
-**Rule:** A metric's unit, labels, aggregation, and lifecycle MUST be documented;
-changing any of them is a contract change.
-
-### OBS-INV-003: Labels are bounded
-
-**Rule:** Metrics MUST NOT use request IDs, prompts, outputs, or other unbounded
-values as labels.
-
-## Safe-change guide
-
-Test registration, label sets, units, lifecycle resets, and representative
-multi-stage emission.

docs/design/module/omni_connector.md

@@ -1,50 +0,0 @@
----
-title: OmniConnector
-kind: module
-status: draft
-owners:
-  - "@princepride"
-  - "@yuanheng-zhao"
-  - "@xuechendi"
-  - "@natureofnature"
-  - "@fake0fan"
-primary_code_paths:
-  - vllm_omni/distributed/omni_connectors/**
-related_code_paths:
-  - vllm_omni/platforms/*/omni_connectors/**
-depends_on:
-  - input_output_modality_contracts.md
-  - vllm_omni_config.md
-validation_paths:
-  - tests/distributed/omni_connectors/**
-upstream_refs:
-  - vllm.distributed
-last_reviewed: 2026-07-16
----
-
-# OmniConnector
-
-OmniConnector defines model-agnostic transport and synchronization contracts
-for data exchanged across stages, processes, devices, and nodes.
-
-## Candidate invariants
-
-### CONNECTOR-INV-001: Connectors transport but do not route
-
-**Rule:** A connector MUST NOT select the next stage or implement model-specific
-execution policy.
-
-### CONNECTOR-INV-002: Producer and consumer contracts agree
-
-**Rule:** Both ends MUST agree on data identity, shape, dtype, placement,
-ownership, and completion semantics.
-
-### CONNECTOR-INV-003: Resources have deterministic cleanup
-
-**Rule:** Connections, buffers, handles, and background work MUST be released on
-normal completion, cancellation, failure, and shutdown.
-
-## Safe-change guide
-
-Test setup, transfer, synchronization, timeout, cancellation, failure, and
-cleanup across every affected backend.

docs/design/module/profiling.md

@@ -1,46 +0,0 @@
----
-title: Profiling
-kind: module
-status: draft
-owners:
-  - "@gcanlin"
-primary_code_paths:
-  - vllm_omni/profiler/**
-  - vllm_omni/diffusion/profiler/**
-related_code_paths:
-  - vllm_omni/entrypoints/**
-depends_on:
-  - observability.md
-validation_paths:
-  - tests/profile/**
-upstream_refs:
-  - vllm.profiler
-last_reviewed: 2026-07-16
----
-
-# Profiling
-
-Profiling owns opt-in instrumentation and trace collection used to locate
-runtime bottlenecks without changing model results or execution policy.
-
-## Candidate invariants
-
-### PROFILE-INV-001: Profiling is opt-in
-
-**Rule:** Expensive trace collection MUST be disabled by default and bounded by
-an explicit start and stop lifecycle.
-
-### PROFILE-INV-002: Instrumentation preserves semantics
-
-**Rule:** Profiling MUST NOT change scheduling decisions, generated outputs, or
-resource ownership beyond documented synchronization overhead.
-
-### PROFILE-INV-003: Trace context is sufficient
-
-**Rule:** Trace events SHOULD identify stage, rank, worker, operation, and
-request where available without embedding user payloads.
-
-## Safe-change guide
-
-Test disabled overhead, repeated start and stop, cleanup, multi-rank trace
-naming, and each supported profiler backend.

docs/design/module/quantization.md

@@ -1,50 +0,0 @@
----
-title: Quantization
-kind: module
-status: draft
-owners:
-  - "@david6666666"
-  - "@Isotr0py"
-  - "@lishunyang12"
-primary_code_paths:
-  - vllm_omni/quantization/**
-  - vllm_omni/diffusion/quantization/**
-related_code_paths:
-  - vllm_omni/platforms/*/quant/**
-depends_on:
-  - model_integration.md
-  - diffusion/diffusion_model_integration.md
-validation_paths:
-  - tests/quantization/**
-  - tests/diffusion/quantization/**
-upstream_refs:
-  - vllm.model_executor.layers.quantization
-last_reviewed: 2026-07-16
----
-
-# Quantization
-
-Quantization owns method registration, checkpoint interpretation, layer
-replacement, and runtime constraints for reduced-precision execution.
-
-## Candidate invariants
-
-### QUANT-INV-001: Checkpoint and runtime methods agree
-
-**Rule:** The selected method MUST validate checkpoint metadata, supported
-layers, device capabilities, and required dependencies before use.
-
-### QUANT-INV-002: Unsupported combinations fail clearly
-
-**Rule:** Quantization MUST NOT silently fall back to a different precision or
-method when a requested combination is unsupported.
-
-### QUANT-INV-003: Numerical validation is required
-
-**Rule:** A new method or layer mapping MUST include correctness comparison
-against a defined reference result.
-
-## Safe-change guide
-
-Test loading, representative layers, metadata, platform guards, and numerical
-tolerances separately from performance.

docs/design/module/stage_runtime.md

@@ -1,126 +0,0 @@
----
-title: Stage Runtime and Replica Lifecycle
-kind: module
-status: draft
-architecture_state: current-plus-in-flight
-owners:
-  - "@tzhouam"
-  - "@fake0fan"
-document_stewards:
-  - "@hsliuustc0106"
-  - "@Gaohan123"
-  - "@david6666666"
-required_reviewers:
-  - "@Sy0307"
-  - "@princepride"
-  - "@chickeyton"
-  - "@yenuo26"
-  - "@NickCao"
-primary_code_paths:
-  - vllm_omni/engine/stage_runtime.py
-  - vllm_omni/engine/stage_pool.py
-  - vllm_omni/engine/stage_client.py
-  - vllm_omni/engine/membership_controller.py
-  - vllm_omni/engine/stage_engine_startup.py
-  - vllm_omni/engine/stage_engine_core_client.py
-  - vllm_omni/engine/stage_engine_core_proc.py
-  - vllm_omni/engine/stage_engine_core_proc_manager.py
-  - vllm_omni/engine/stage_init_utils.py
-related_code_paths:
-  - vllm_omni/distributed/omni_coordinator/**
-  - vllm_omni/distributed/ray_utils/**
-  - vllm_omni/diffusion/inline_stage_diffusion_client.py
-  - vllm_omni/diffusion/stage_diffusion_client.py
-  - vllm_omni/diffusion/stage_diffusion_proc.py
-  - vllm_omni/config/**
-depends_on:
-  - input_output_modality_contracts.md
-  - error_contracts.md
-  - omni_connector.md
-validation_paths:
-  - tests/engine/test_async_omni_engine_stage_init.py
-  - tests/engine/test_membership_controller.py
-  - tests/engine/test_single_stage_mode.py
-  - tests/engine/test_stage_engine_core_client.py
-  - tests/engine/test_stage_engine_core_proc.py
-  - tests/diffusion/test_inline_stage_diffusion_client.py
-  - tests/diffusion/test_stage_diffusion_proc.py
-  - tests/distributed/omni_coordinator/**
-upstream_refs:
-  - vllm.v1.engine.core_client.AsyncMPClient
-  - vllm.v1.engine.core_client.DPLBAsyncMPClient
-  - vllm.v1.engine.core.EngineCoreProc
-  - vllm.v1.engine.utils.CoreEngineProcManager
-  - vllm.v1.engine.coordinator.DPCoordinator
-invariant_namespace: STAGE-INV
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# Stage runtime and replica lifecycle
-
-The stage runtime turns a logical stage definition into ready local or
-distributed replicas and provides the lifecycle boundary used by the
-orchestrator.
-
-## Contract status
-
-This document describes the current `StageRuntime`, `StagePool`, stage-client,
-and stage-process layout. The unified LLM/diffusion direction in
-[#5441](https://github.com/vllm-project/vllm-omni/pull/5441) remains in flight
-and is not presented as current behavior.
-
-## Ownership boundary
-
-This document owns local and distributed stage placement, startup and
-readiness, one pool per logical stage, replica identity and membership,
-selection and affinity, stage-client/process lifecycle, liveness, draining,
-and shutdown.
-
-It does not own cross-stage request policy, model scheduler policy, connector
-transfer semantics, configuration precedence, or public error rendering.
-
-## Candidate invariants
-
-These identifiers are proposals while the document is `draft`.
-
-### STAGE-INV-001: The runtime owns replica lifecycle
-
-**Rule:** Orchestration MUST acquire stage capacity through the stage runtime
-and MUST NOT construct or retire backend replicas directly.
-
-### STAGE-INV-100: Affinity uses stable replica identity
-
-**Rule:** Follow-up operations for a request with replica affinity MUST resolve
-the same logical replica until the request terminates or the replica is
-declared lost.
-
-### STAGE-INV-200: Shutdown is idempotent
-
-**Rule:** Repeated shutdown or cleanup signals MUST NOT leak stage clients,
-process managers, or membership registrations.
-
-## Invariant namespace
-
-`STAGE-INV` reserves `001-099` for placement and ownership, `100-199` for
-replica identity/readiness/affinity, `200-299` for liveness loss, draining and
-cleanup, and `300-399` for local/distributed and upstream compatibility.
-Numbers become append-only after normative promotion.
-
-## Safe-change guide
-
-Exercise local and distributed startup, readiness, replica selection,
-affinity, membership changes, abort routing, process failure, draining, and
-repeated shutdown. Cross-stage policy changes belong in
-`engine_orchestration.md`.
-
-## Promotion gate
-
-- Reconcile terminology and ownership after #5441 reaches a final state.
-- Add or identify a dedicated StageRuntime/StagePool lifecycle suite; current
-  evidence is distributed across the cited tests.
-- Demonstrate affinity preservation for update, interaction, and abort paths.
-- Demonstrate membership loss and repeated shutdown without leaked clients or
-  processes.
-- Obtain approval from a technical owner and an independent validation
-  reviewer.

docs/design/module/vllm_omni_config.md

@@ -1,56 +0,0 @@
----
-title: vLLM-Omni Configuration
-kind: module
-status: draft
-architecture_state: deferred-pending-refactor
-ownership_status: provisional
-owners:
-  - "@lishunyang12"
-  - "@alex-jw-brooks"
-primary_code_paths:
-  - vllm_omni/config/**
-  - vllm_omni/deploy/**
-  - vllm_omni/model_executor/stage_configs/**
-related_code_paths:
-  - vllm_omni/platforms/*/stage_configs/**
-depends_on: []
-validation_paths:
-  - tests/config/**
-upstream_refs:
-  - vllm.config
-last_reviewed: 2026-08-07
-last_verified_commit: 3d7fc3b9ba3cac88d579d4dc35b78b0b641675fc
----
-
-# vLLM-Omni configuration
-
-This page intentionally records discovery paths only. Substantive
-configuration ownership and contract work is deferred until the active
-configuration refactoring has settled.
-
-## Deferred contract scope
-
-The current code and tests remain authoritative for parsing, defaults,
-overrides, deployment topology, stage construction, and runtime projection.
-This draft does not define a stable precedence model, environment-variable
-capture rule, canonical configuration object, or invariant namespace.
-
-The PR review proposal to capture environment-derived values in configuration
-objects at initialization is an open design question, not a current
-invariant.
-
-## Safe-change guide while deferred
-
-Changes should trace every affected producer to its runtime consumer and test
-the structured, legacy, CLI, and deployment paths that are actually supported.
-Do not infer a stable contract from this placeholder page.
-
-## Promotion gate
-
-- Finish or stabilize the configuration refactor and identify the canonical
-  runtime configuration representation.
-- Confirm technical owners, primary path exceptions, and validation owners.
-- Document parsing and override precedence, including when environment values
-  are captured.
-- Allocate an invariant namespace only after those behaviors are enforced and
-  owner-approved.

docs/design/qwen3_omni_tts_performance_optimization.md

@@ -419,7 +419,8 @@ Notes:
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
   --omni \
-  --port 8091
+  --port 8091 \
+  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
 ```
 
 #### 3) Key config knobs
@@ -493,7 +494,7 @@ The default config (`qwen3_tts.yaml`) enables the full optimization stack:
 vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
   --omni \
   --port 8000 \
-  --no-async-chunk
+  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_no_async_chunk.yaml
 ```
 
 #### 3) Key config knobs

docs/getting_started/installation/README.md

@@ -1,8 +1,5 @@
 # Installation
 
-!!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.
-
 vLLM-Omni supports the following hardware platforms:
 
 - [GPU](gpu.md)

docs/getting_started/installation/gpu/cuda.inc.md

@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]
 
-vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
+vLLM-Omni depends vLLM. So please follow instructions below mainly for vLLM.
 
 !!! note
     PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -124,7 +124,7 @@ If you want to specify the base vLLM version:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
   -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.22.1 \
   -t vllm-omni-cuda .
 ```
 

docs/getting_started/installation/npu/npu.inc.md

@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
 
 ```bash
 # vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
+# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.25.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
+# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.25.0-a3
 docker run --rm \
     --name vllm-omni-npu \
     --shm-size=1g \
@@ -28,11 +28,11 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v /root/.cache:/root/.cache \
     -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
+    -it quay.io/atlas-ci/vllm-ascend:0.25.0-a3 bash
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.25.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -53,20 +53,16 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)
 
 ```bash
-# Pin vLLM and vLLM-Ascend to the v0.26 release line
+# Pin vLLM version to 0.18.0
-git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
+git clone -b v0.18.0 https://github.com/vllm-project/vllm.git
-cd vllm
 VLLM_TARGET_DEVICE=empty pip install -v -e .
-cd ..
 
-git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
+git clone -b v0.18.0rc1 https://github.com/vllm-project/vllm-ascend.git
-cd vllm-ascend
 pip install -v -e .
-cd ..
 
 # Install vLLM-Omni from the latest main branch
 git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
+cd /vllm-workspace/vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -88,8 +84,8 @@ Supported images as following.
 Here's an example deployment command that has been verified on 4 x NPUs:
 
 ```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
+# Atlas A2: quay.io/ascend/vllm-omni:v0.25.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
+# Atlas A3: quay.io/ascend/vllm-omni:v0.25.0-a3
 docker run --rm \
     --name vllm-omni-a3 \
     --shm-size=4g \
@@ -114,7 +110,7 @@ docker run --rm \
     -v /etc/ascend_install.info:/etc/ascend_install.info \
     -v ~/.cache:/root/.cache \
     -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
+    -it quay.io/ascend/vllm-omni:v0.25.0-a3 bash
 ```
 
 !!! tip
@@ -126,9 +122,9 @@ Or build IMAGE from **source code**:
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 # A2
-# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
+docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
 # A3
-docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
+# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -87,8 +87,9 @@ if __name__ == "__main__":
 
 !!! info
 
-    For diffusion request batching, step execution, and streaming controls, see
+    For diffusion request-level batching controls such as `max_num_seqs` and
-    [Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
+    `request_batch_max_wait_ms`, see
+    [Request-Level Batching](../user_guide/diffusion/request_batching.md).
 
 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
 

docs/models/supported_models.md

@@ -44,7 +44,6 @@ th {
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
 | `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
-| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
 | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |

docs/serving/speech_api.md

@@ -276,42 +276,20 @@ Client -> Server:
 
 | Message | Description |
 |---------|-------------|
-| `{"type": "session.config", ...}` | Session configuration (first message; may be resent between utterances to change it) |
+| `{"type": "session.config", ...}` | Session configuration (sent once, first message) |
 | `{"type": "input.text", "text": "..."}` | Text chunk |
-| `{"type": "input.done"}` | End of utterance: flushes the buffer and keeps the connection open |
+| `{"type": "input.done"}` | End of input, flushes remaining buffer |
-| `{"type": "session.close"}` | End of connection |
 
 Server -> Client:
 
 | Message | Description |
 |---------|-------------|
-| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
+| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
-| `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
+| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
-| `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete |
+| `{"type": "session.done", "total_sentences": N}` | Session complete |
 | `{"type": "error", "message": "..."}` | Non-fatal error |
 
-### Flushing vs. Closing
-
-`input.done` is a flush, not a disconnect. The server synthesizes the buffered
-text, emits `session.done`, and then waits on the same connection for the next
-utterance, so a client that speaks repeatedly (for example one driven by an
-upstream LLM) pays the WebSocket handshake once instead of once per utterance.
-
-* The session config is sticky. Send `input.text` again straight after
-  `session.done` to reuse it, or send another `session.config` first to change
-  voice, format, or reference audio. A `session.config` sent while text is
-  still buffered is rejected so no pending input is silently dropped.
-* An utterance is the flush unit, not a linguistic one: it is whatever text was
-  buffered when `input.done` arrived, of any length, synthesized as one request.
-  `utterance_index` counts those flushes across the connection, so it tells you
-  which `input.done` a frame belongs to. `sentence_index` counts within one
-  flush and so pairs with `total_sentences`, which means every utterance reports
-  `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer).
-* End the connection with `session.close`, or by closing the socket. An idle
-  connection is still closed after the server's idle timeout, which now also
-  applies to the gap between utterances.
-
 ### Session Config Parameters
 
 All REST API parameters are supported, plus:

docs/source/architecture/qwen3-omni-async-output-step-gap.svg

@@ -1,150 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="1024" height="600" viewBox="0 0 1024 600" role="img" aria-labelledby="title desc">
-  <title id="title">Qwen3-Omni Talker step gap before and after asynchronous output materialization</title>
-  <desc id="desc">Before the optimization, synchronous D2H copy, per-request payload construction, streaming payload handoff, and connector-signal drain separate consecutive Talker forward and sample steps by about 2.8 milliseconds. After the optimization, that Omni output work runs on a background builder while the next Talker step executes, leaving a boundary of about 41 microseconds for snapshot, D2H enqueue, state update, and sampled-token feedback.</desc>
-  <defs>
-    <marker id="time-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#64748b"/>
-    </marker>
-    <marker id="red-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#b91c1c"/>
-    </marker>
-    <marker id="red-start" viewBox="0 0 10 10" refX="2" refY="5" markerWidth="5" markerHeight="5" orient="auto">
-      <path d="M 10 0 L 0 5 L 10 10 z" fill="#b91c1c"/>
-    </marker>
-    <marker id="blue-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#2563eb"/>
-    </marker>
-    <style>
-      .canvas { fill: #ffffff; stroke: #e5e7eb; }
-      .title { font: 700 22px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
-      .subtitle { font: 500 13px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .phase { font: 700 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1f2937; }
-      .phase-sub { font: 500 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .lane { font: 600 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #374151; }
-      .block-title { font: 700 10px system-ui, -apple-system, Segoe UI, sans-serif; letter-spacing: .25px; fill: #ffffff; }
-      .block-op { font: 500 9.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
-      .axis { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #94a3b8; }
-      .measure { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #b91c1c; }
-      .inset-title { font: 700 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
-      .inset-step { font: 600 10px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
-      .note-title { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1e3a8a; }
-      .note-text { font: 500 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
-      .stat-cap { font: 600 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .stat-big { font: 800 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
-      .stat-sub { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .before-card { fill: #eef4ff; stroke: #c7dbf6; }
-      .after-card { fill: #eafaf0; stroke: #bfe7cf; }
-      .talker { fill: #16a34a; }
-      .sync-output { fill: #ea580c; }
-      .async-output { fill: #2563eb; }
-      .zoom-card { fill: #fff7f7; stroke: #f3caca; }
-      .note-card { fill: #eff6ff; stroke: #bfdbfe; }
-      .stat-blue { fill: #eef4ff; stroke: #c7dbf6; }
-      .stat-green { fill: #eafaf0; stroke: #bfe7cf; }
-      .stat-gray { fill: #f4f5f7; stroke: #e2e5ea; }
-      .guide { stroke: #e2e8f0; stroke-width: 1; }
-      .time-line { stroke: #64748b; stroke-width: 1.3; fill: none; marker-end: url(#time-arrow); }
-      .measure-line { stroke: #b91c1c; stroke-width: 1.4; marker-start: url(#red-start); marker-end: url(#red-end); }
-      .zoom-lead { stroke: #dc2626; stroke-width: 1.15; stroke-dasharray: 4 3; fill: none; }
-      .handoff { stroke: #2563eb; stroke-width: 1.3; stroke-dasharray: 3 2; fill: none; marker-end: url(#blue-end); }
-    </style>
-  </defs>
-
-  <rect x="14" y="14" width="996" height="572" rx="14" class="canvas"/>
-
-  <text x="44" y="52" class="title">Async output: what moves off the Talker decode path</text>
-  <text x="44" y="74" class="subtitle">Qwen3-Omni Talker stage &#183; consecutive decode steps in a fixed profiler window</text>
-
-  <!-- ===================== BEFORE ===================== -->
-  <rect x="44" y="94" width="144" height="110" rx="10" class="before-card"/>
-  <text x="60" y="120" class="phase">Before</text>
-  <text x="60" y="139" class="phase-sub">Omni output built</text>
-  <text x="60" y="156" class="phase-sub">synchronously on the</text>
-  <text x="60" y="173" class="phase-sub">AR critical path</text>
-
-  <text x="206" y="122" class="lane">AR path</text>
-  <line x1="300" y1="88" x2="300" y2="210" class="guide"/>
-
-  <rect x="300" y="104" width="190" height="54" rx="5" class="talker"/>
-  <text x="395" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
-  <text x="395" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <rect x="490" y="104" width="250" height="54" rx="5" class="sync-output"/>
-  <text x="615" y="120" text-anchor="middle" class="block-title">SYNC OMNI OUTPUT &#183; STEP N</text>
-  <text x="615" y="138" text-anchor="middle" class="block-op">D2H &#8594; per-request slice / payload build</text>
-  <text x="615" y="152" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
-
-  <rect x="740" y="104" width="220" height="54" rx="5" class="talker"/>
-  <text x="850" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
-  <text x="850" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <line x1="490" y1="174" x2="740" y2="174" class="measure-line"/>
-  <text x="615" y="194" text-anchor="middle" class="measure">~2.8 ms GPU gap: output work is inline</text>
-
-  <!-- ===================== AFTER ===================== -->
-  <rect x="44" y="218" width="144" height="140" rx="10" class="after-card"/>
-  <text x="60" y="244" class="phase">After</text>
-  <text x="60" y="263" class="phase-sub">Only step feedback</text>
-  <text x="60" y="280" class="phase-sub">stays inline; payload</text>
-  <text x="60" y="297" class="phase-sub">work runs in the</text>
-  <text x="60" y="314" class="phase-sub">background</text>
-
-  <text x="206" y="254" class="lane">Talker GPU</text>
-  <text x="206" y="326" class="lane">Output builder</text>
-  <line x1="300" y1="212" x2="300" y2="370" class="guide"/>
-
-  <rect x="300" y="226" width="235" height="54" rx="5" class="talker"/>
-  <text x="418" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
-  <text x="418" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <rect x="540" y="226" width="235" height="54" rx="5" class="talker"/>
-  <text x="658" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
-  <text x="658" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <path d="M 535 280 L 548 298" class="handoff"/>
-  <rect x="540" y="298" width="360" height="58" rx="5" class="async-output"/>
-  <text x="720" y="315" text-anchor="middle" class="block-title">BACKGROUND OMNI OUTPUT &#183; STEP N</text>
-  <text x="720" y="334" text-anchor="middle" class="block-op">wait for D2H &#8594; per-request slice / payload build</text>
-  <text x="720" y="349" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
-
-  <line x1="300" y1="366" x2="930" y2="366" class="time-line"/>
-  <text x="922" y="382" text-anchor="end" class="axis">time</text>
-
-  <!-- leads from the real 41 us boundary to the magnified inset -->
-  <line x1="535" y1="280" x2="485" y2="424" class="zoom-lead"/>
-  <line x1="540" y1="280" x2="525" y2="424" class="zoom-lead"/>
-
-  <!-- ===================== 41 µs BOUNDARY ===================== -->
-  <rect x="300" y="386" width="400" height="92" rx="10" class="zoom-card"/>
-  <text x="316" y="404" class="inset-title">Zoom &#183; Talker step boundary (magnified)</text>
-
-  <rect x="320" y="416" width="165" height="26" rx="4" class="talker"/>
-  <rect x="525" y="416" width="155" height="26" rx="4" class="talker"/>
-  <text x="403" y="433" text-anchor="middle" class="inset-step">Talker step N</text>
-  <text x="603" y="433" text-anchor="middle" class="inset-step">Talker step N+1</text>
-
-  <line x1="485" y1="452" x2="525" y2="452" class="measure-line"/>
-  <text x="505" y="469" text-anchor="middle" class="measure">~41 &#181;s</text>
-
-  <rect x="720" y="386" width="240" height="92" rx="10" class="note-card"/>
-  <text x="736" y="405" class="note-title">Inline at the ~41 &#181;s boundary</text>
-  <text x="736" y="424" class="note-text">&#8226; eager Talker state update</text>
-  <text x="736" y="441" class="note-text">&#8226; output snapshot + D2H enqueue</text>
-  <text x="736" y="458" class="note-text">&#8226; sampled-token feedback</text>
-
-  <!-- ===================== SUMMARY ===================== -->
-  <rect x="44" y="494" width="296" height="70" rx="9" class="stat-blue"/>
-  <text x="60" y="514" class="stat-cap">Step-to-step gap</text>
-  <text x="60" y="538" class="stat-big">2.8 ms &#8594; 41 &#181;s</text>
-  <text x="60" y="555" class="stat-sub">~68&#215; tighter</text>
-
-  <rect x="356" y="494" width="296" height="70" rx="9" class="stat-green"/>
-  <text x="372" y="514" class="stat-cap">Talker GPU</text>
-  <text x="372" y="538" class="stat-big">next step starts immediately</text>
-  <text x="372" y="555" class="stat-sub">decode workers stay busy</text>
-
-  <rect x="668" y="494" width="312" height="70" rx="9" class="stat-gray"/>
-  <text x="684" y="514" class="stat-cap">Background builder</text>
-  <text x="684" y="538" class="stat-big">materializes Omni output</text>
-  <text x="684" y="555" class="stat-sub">payload work overlaps the next Talker step</text>
-</svg>

docs/user_guide/diffusion/attention_backends.md

@@ -14,8 +14,8 @@ The full set of backends and their platform defaults is in the **Backend Options
 
 | Value | Notes |
 |---|---|
-| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). Dense BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Packed paths can provide `cu_seqlens` directly. Supports optional **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
+| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Supports **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
-| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. |
+| `FLASH_ATTN` | Wraps FlashAttention 2. Default on Hopper / Ada / Ampere when `flash-attn` is installed. |
 | `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
 | `FLASHINFER_ATTN` | Calls FlashInfer's dense `single_prefill_with_kv_cache` directly with `custom_mask` for non-causal masked attention. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer`. |
 | `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
@@ -23,7 +23,6 @@ The full set of backends and their platform defaults is in the **Backend Options
 | `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. |
 | `FLASH_ATTN_HUB` | FlashAttention 2 from HuggingFace `kernels` library. Useful for train/rollout alignment. |
 | `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs. |
-| `RAINFUSION_ATTN` | MindIE-SD **RainFusion** block-sparse video attention — see [below](#rainfusion_attn-backend-and-block-sparse-video-attention). Ascend NPU only; requires `mindiesd`. Delegates to `FLASH_ATTN` for anything that is not a packed video sequence. |
 
 
 ## Configuration
@@ -70,9 +69,8 @@ vllm-omni serve <model> \
     --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
 ```
 
-A backend that needs configuration exposes it as a typed field on the spec:
+A backend that needs configuration exposes it as a typed field on the spec. Today the only
-`TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)) and
+one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):
-`RAINFUSION_ATTN`'s RainFusion block (see [below](#rainfusion_attn-backend-and-block-sparse-video-attention)):
 
 ```bash
 --diffusion-attention-config.default.backend TRTLLM_ATTN \
@@ -105,7 +103,7 @@ A plain dict is also accepted and normalized to `AttentionConfig`.
 
 Auto-route preference, in order:
 
-1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model declares a compatible packed/mask-free path**
+1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model is mask-free** (see below)
 2. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels)
 3. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5
 4. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel
@@ -113,7 +111,7 @@ Auto-route preference, in order:
 
 `TRTLLM_ATTN` is skipped on workstation Blackwell (sm_120 / sm_121) and for any `head_dim != 128`, so those GPUs keep the `CUDNN_ATTN` route described below.
 
-`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell for compatible packed/mask-free pipelines. Workstation Blackwell (sm_120 / sm_121) and pipelines that require attention masks retain their normal fallback.
+`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell because it is the only backend that can enable Skip-Softmax, not because its dense kernel is faster — dense, the two are comparable. It cannot honor an attention mask, so the auto-default only applies to pipelines verified mask-free (the Wan family); mask-using pipelines keep `CUDNN_ATTN`. `TRTLLM_ATTN` stays available everywhere via explicit `--diffusion-attention-backend TRTLLM_ATTN` (which raises clearly if a mask is received).
 
 The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route.
 
@@ -169,117 +167,6 @@ sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `T
 Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting
 `TRTLLM_ATTN` raises rather than silently degrading.
 
-## TRTLLM_ATTN SAGE Quantization
-
-On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen
-FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run
-in low precision. Enable it through the typed `quant` block on the attention spec:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). |
-| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). |
-| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). |
-
-V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so
-neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels.
-
-The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the
-fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),
-`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),
-`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'
-```
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16),
-    ),
-)
-```
-
-**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel
-availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**
-(B300); `int8` QK kernels are compiled for **SM100 only**.
-
-## RAINFUSION_ATTN Backend and Block-Sparse Video Attention
-
-`RAINFUSION_ATTN` runs MindIE-SD's RainFusion (`rf_v2`) kernel on Ascend NPU. It pools each
-128-token block of keys, ranks them per query block, and computes attention against only the
-top-scoring ones. Video tokens are rearranged into `(t, h, w)` order first, so the kept blocks are
-spatiotemporal neighbours rather than arbitrary rows of the packed sequence.
-
-Only the video segment is sparsified. The prefix rows (text, visual conditions, audio) and the
-first-frame blocks are always kept dense, which is why the realized sparsity is lower than the
-nominal `sparsity` you configure. Anything the kernel cannot handle — a warmup denoise step, an
-exempt layer, a sequence with no published video geometry, a video segment under 32 blocks —
-delegates to `FLASH_ATTN`, so a model can select this backend unconditionally.
-
-Enable it through the typed `block_sparse` block on the attention spec, shared by every
-block-sparse backend:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `sparsity` | finite, `[0, 1]` | Nominal fraction of key blocks dropped per query block. `0` disables sparsity. Defaults to `0.8`. |
-| `start_step` | `≥ 0` | Keep the first N denoise steps dense. Layout is decided early, so these steps dominate structural fidelity. |
-| `skip_layers` | index selector, e.g. `"0-3,38"` | DiT blocks that always stay dense. |
-
-```bash
-vllm-omni serve MiniMaxAI/MiniMax-H3 \
-  --diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN",
-      "block_sparse": {"sparsity": 0.8, "start_step": 0}}}'
-```
-
-Programmatically the same block is a typed `BlockSparseSpec` (values validated at construction):
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="RAINFUSION_ATTN",
-        block_sparse=BlockSparseSpec(sparsity=0.8, start_step=0, skip_layers="0-1"),
-    ),
-)
-```
-
-**Tune in the order `start_step` → `sparsity` → `skip_layers`.** Raise `start_step` first: it is the
-cheapest way to recover structure, because the early high-noise steps decide global layout while the
-later steps only refine texture. Then walk `sparsity` up to your quality bar. Reach for
-`skip_layers` last, once an A/B against dense at the same seed points at specific blocks.
-
-Requires Ascend NPU with `mindiesd`; selecting it on any other platform raises. It is also
-incompatible with ring sequence parallelism, since `rf_v2` needs the whole key sequence to rank
-blocks — use Ulysses SP (`ring_degree=1`).
-
-### Which geometries run sparse
-
-Sparsity is only applied when the video segment is a **multiple of 128 rows**, where the row count
-is `latent_t × (height / 32) × (width / 32)`. Otherwise `rf_v2`'s block mask and the kernel's own
-tiling disagree on the block count, and the block straddling the seam mixes video and prefix rows
-that selection may then drop. Any resolution still runs — an unaligned geometry falls back to dense
-attention and logs `RAINFUSION_ATTN staying dense` with the row count it computed — but it gets no
-speedup, so pick an aligned geometry when you want one.
-
-Alignment is necessary, not sufficient, for good quality. `rf_v2` groups video positions into 8x8
-spatial tiles (two tiles fill one 128-row block), which is what makes a selected block a compact
-patch of the frame. When the latent `h` or `w` is not a multiple of 8, the leftover rows or columns
-are peeled off and appended as a flat run instead of being tiled, so they get pooled with spatially
-distant positions and selection can no longer rank them meaningfully. This is silent, and invisible
-to a `sparsity=0` check, because a fully populated mask does not care how blocks are grouped.
-
-"Multiple of 8" on the latent grid means **width and height that are multiples of 256**. Aligned
-resolutions off that grid still run sparse; they just lose more fidelity at the same `sparsity`,
-which you can buy back with `start_step`.
-
 ## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)
 
 Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
@@ -306,17 +193,9 @@ DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/te
     --model Lightricks/LTX-2 ...
 ```
 
-### FlashAttention-4 on Blackwell
+### FA4 not yet integrated
 
-Install the optional CUDA 13 extra to use the CuTe-based FA4 path:
+FlashAttention-4 (released March 2026) targets Blackwell natively and reportedly beats cuDNN by ~20% on B200. As of this writing the `flash-attn-4 4.0.0b10` wheel crashes with `AttributeError: 'NoneType' object has no attribute '_trait'` during JIT on sm_120. Not yet wired into vLLM-Omni; revisit when stable lands.
-
-```bash
-pip install 'vllm-omni[fa4]'
-```
-
-`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2
-when it is unavailable. Version `4.0.0b18` is required; earlier beta
-wheels had known JIT failures on Blackwell.
 
 ## Choosing a Backend Manually
 
@@ -474,3 +353,64 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_t
     --tensor-parallel-size 2 \
     --output outputs/hv15_sage3.mp4
 ```
+
+### Mixed backends across roles
+
+Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --diffusion-attention-config.per_role.self.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_mixed.mp4
+```
+
+### Compare against FlashAttention
+
+Unset the backend override, or explicitly use `FLASH_ATTN`:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --height 704 --width 1280 --num-frames 49 \
+    --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_fa3.mp4
+```
+
+## Validation Guidance
+
+Don't assume a faster attention backend is numerically interchangeable with `TORCH_SDPA`.
+
+Always compare:
+
+- End-to-end runtime
+- Diffusion-stage runtime (`add_req_and_wait` line in DiffusionEngine.step breakdown)
+- Output quality against a known-good baseline (CLIP similarity, frame-level diff, or visual review)
+
+At minimum, keep the same:
+
+- model
+- prompt
+- seed
+- resolution
+- frame count / step count
+- parallel config (TP / CFG-parallel / Ulysses degrees)
+
+## Reproducing the Benchmark Table
+
+The end-to-end numbers above were collected by running `text_to_video.py` /
+`text_to_image.py` with the same prompt and seed while varying
+`DIFFUSION_ATTENTION_BACKEND`. For a quick kernel-level comparison of the
+backends without loading a model:
+
+```bash
+python benchmarks/diffusion/bench_attention_backends.py --preset hv15
+```
+
+It runs all three BF16 backends on representative DiT attention shapes and
+prints a ranking table at the end.

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -179,145 +179,7 @@ class Flux2Transformer2DModel(nn.Module):
 - Support single GPU only for now
 
 
-## Distributed Layerwise Offloading
+### Implementation Notes
-
-### How It Works
-
-Distributed layerwise offloading extends single-GPU layerwise offloading to
-multi-device deployments. In the default mode, each rank stores only
-**1/dp_size** of the model weights in host memory; full layer weights are
-reconstructed at runtime via **AllGather** on a dedicated communication stream,
-overlapped with computation via a fixed double-buffer scheme. The
-`--dlo-no-use-allgather` mode instead streams standard-loader rank-local
-weights independently and does not shard weights across ranks.
-
-For the implementation design and compatibility matrix, see
-[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md).
-
-**Key features:**
-- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights,
-  reconstructed per-layer via `all_gather_into_tensor`
-- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any
-  time, regardless of model size
-- **DP multi-concurrency**: N concurrent requests processed in parallel
-  (AllGather only gathers weight shards, which are request-independent)
-- **mmap weight loading**: weights loaded as mmap views pointing to shared OS
-  page cache, eliminating O(dp_size × model_size) RSS during model creation
-- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU
-  (CANN/HCCL) via vLLM-Omni's platform abstraction
-
-**Execution Flow (per device):**
-
-```
-Compute Stream:  [Layer N]          [Layer N+1]          [Layer N+2]
-H2D Stream:      [H2D Shard N+1]   [H2D Shard N+2]
-AllGather:       [AG N+1]          [AG N+2]
-Slot Usage:      Slot0: Layer N    Slot1: Layer N+1
-```
-
-### Usage
-
-**CLI:**
-```bash
-# 4× GPU/NPU with AllGather (recommended)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4
-
-# Without DLO AllGather (each rank streams standard-loader rank-local weights)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4 \
-  --dlo-no-use-allgather
-
-# With SP instead of DP (long sequences)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --usp 4
-```
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(
-    model="/path/to/model",
-    enable_distributed_layerwise_offload=True,
-    dlo_use_allgather=True,  # default
-)
-```
-
-### CLI Flags
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
-| `--data-parallel-size N` | Number of DP ranks; DLO weight-sharding group when AllGather is enabled | `1` |
-| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` |
-| `--dlo-no-use-allgather` | Use standard-loader rank-local weights; no DLO sharding or AllGather | `false` |
-
-### How model weights are loaded (mmap path)
-
-When DLO + AllGather is active, the offloader:
-
-1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`)
-   from the normally-created transformer
-2. **Converts to meta device** via `to_empty(device="meta")`, releasing random
-   initialization weights
-3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`,
-   which return views into the OS page cache (shared across all ranks, 0 RSS)
-4. **Calls `post_load_weights()`** to apply model-specific dtype conversions
-   (e.g. Cosmos3's `time_embedder` → FP32)
-5. **Restores non-persistent buffers** from saved copies
-
-This approach requires **zero model-specific code changes** — no pipeline or
-transformer modifications are needed.
-
-### OffloadPlan (declarative topology metadata)
-
-Models can optionally declare an `OffloadPlan` class variable to provide
-topology metadata (block attribute names, submodules to offload) without
-any offload-specific logic:
-
-```python
-from vllm_omni.diffusion.offloader import OffloadPlan
-
-class MyPipeline(nn.Module):
-    _dit_modules = ["transformer"]
-    _offload_plan = OffloadPlan(
-        block_attrs={"transformer": ("blocks",)},
-        offload_submodules={"context_encoder": "layers"},
-    )
-```
-
-When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs`
-and heuristic attribute search.  This is backward-compatible — existing models
-work without any changes.
-
-### DP Multi-concurrency
-
-When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches
-up to `dp_size` requests per denoise step.  Each DP rank processes a different
-request while AllGather synchronizes only weight shards (request-independent).
-
-**Requirements for concurrent requests:**
-- `num_inference_steps` must be specified explicitly (None is not allowed)
-- All concurrent requests must have the same `num_inference_steps` value
-- These constraints exist because AllGather is a collective that requires
-  every rank to participate at each denoise step
-
-### Limitations
-- Online quantization (FP8) is incompatible with mmap loading — falls back to
-  regular loading only when the model is outside the DLO+AllGather mmap path;
-  online quantization with DLO+AllGather is rejected. Use
-  `--dlo-no-use-allgather` or disable online quantization.
-- Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path because TP-aware
-  weight-loader callbacks are bypassed. The no-AllGather path retains the
-  standard TP loader and is experimental.
-- HSDP + AllGather is rejected because it would double-shard parameters. HSDP
-  with no-AllGather is accepted at configuration level, but has limited
-  end-to-end validation.
-- `num_inference_steps=None` is not allowed in DP multi-concurrency mode
 
 **Module Discovery**
 
@@ -346,10 +208,7 @@ OffloadBackend (base class)
 ├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap)
 │                              (delegates to a pipeline's enable_omni_model_cpu_offload
 │                               for split models like Cosmos3)
-├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
+└── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
-│                          (single-GPU, full weights on host)
-└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook
-                                         (multi-GPU, sharded AllGather or rank-local mode)
 ```
 
 Factory function `get_offload_backend()` selects the appropriate backend based on
@@ -362,28 +221,20 @@ reasoner/generator components inside the model forward pass.
 
 ## Supported Models
 
-| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) |
+| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Blocks Attrs (Layerwise specific) |
-|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------|
+|--------------|----------------|-----------|---------------------|-------------------|-----------------------------------|
-| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` |
+| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` |
-| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` |
+| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` |
-| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` |
+| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` |
-| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` |
+| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
-| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
+| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
-| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
+| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
-| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` |
+| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` |
-| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |
-| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` |
 
 **Notes:**
 - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
 - Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
-- Distributed Layerwise Offloading reuses the layerwise block topology, but the
-  model and parallelism combination still needs validation. AllGather mode
-  additionally requires a compatible checkpoint/mmap layout; no-AllGather mode
-  retains the standard loader's rank-local tensors. See the
-  [DLO design](../../design/feature/distributed_layerwise_offload.md) and
-  [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
-  for usage examples.

docs/user_guide/diffusion/execution_modes.md

@@ -1,252 +0,0 @@
-# Diffusion Execution Modes
-
-vLLM-Omni supports complete-request execution and step-wise diffusion
-execution. Both modes use the same asynchronous engine output stream; the
-configuration controls how work is scheduled and whether intermediate outputs
-are exposed.
-
-## Choose a Mode
-
-| Goal | CLI configuration |
-|---|---|
-| Serial request execution | `--max-num-seqs 1` |
-| Fused request-level batching | `--max-num-seqs N` |
-| Single-request step execution | `--step-execution --max-num-seqs 1` |
-| Step-wise continuous batching | `--step-execution --max-num-seqs N` |
-| Chunked diffusion output | `--diffusion-streaming-output` |
-
-`N` must be greater than `1` to allow batching. The selected pipeline must
-support request-level batching or batched-step execution as appropriate;
-single-request step support alone is not sufficient for step-wise continuous
-batching.
-
-## Request Execution
-
-Request execution is the default when `--step-execution` is omitted. The
-pipeline performs a complete `forward()` for each scheduler wave.
-
-### Serial Requests
-
-Use the conservative serial path with any request-mode pipeline:
-
-```bash
-vllm serve MODEL --omni \
-  --port 8091 \
-  --max-num-seqs 1
-```
-
-### Request-Level Batching
-
-Set `max_num_seqs` above one to combine compatible independent requests into a
-single pipeline forward:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4
-```
-
-For bursty traffic, add a small admission window:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4 \
-  --request-batch-max-wait-ms 20
-```
-
-`--request-batch-max-wait-ms 0` is the default. A nonzero value can improve
-batch formation but adds up to that much latency before a new scheduler wave.
-It has no effect in step mode.
-
-Each prompt remains a separate logical request. Do not submit a top-level list
-as one packed prompt. Concurrent serving requests are batched internally when
-their shapes, guidance settings, output counts, and LoRA settings are
-compatible.
-
-Only pipelines that declare request-batch support accept
-`max_num_seqs>1` in request mode. Unsupported pipelines fail during engine
-initialization; use `max_num_seqs=1` instead.
-
-## Step Execution
-
-Step execution lets the scheduler advance and abort requests between denoise
-steps:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 1
-```
-
-Set a larger capacity to allow compatible requests to share denoise waves:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
-```
-
-Step continuous batching is experimental. Start with `max_num_seqs=1` when
-validating a model or debugging correctness, then increase it for
-multi-request throughput.
-
-Step execution is capability-based, not a generic switch for every diffusion
-model. Qwen-Image supports step-wise continuous batching. HunyuanImage3 also
-supports it, but only when its resolved self-attention backend is `TORCH_SDPA`;
-set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or configure
-`diffusion_attention_config.default.backend=TORCH_SDPA` before using
-`--max-num-seqs >1`. See the
-[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
-for its validated configuration. Helios supports single-request step execution only: use
-`--step-execution --max-num-seqs 1` for Helios. Consult the selected pipeline's
-documentation and source for the latest support status.
-
-## Streaming Output
-
-Use `--diffusion-streaming-output` for a pipeline that can produce intermediate
-diffusion outputs:
-
-```bash
-vllm serve BestWishYsh/Helios-Distilled --omni \
-  --port 8000 \
-  --diffusion-streaming-output
-```
-
-Streaming output requires step execution. If
-`--diffusion-streaming-output` is set without `--step-execution`, the engine
-enables step execution automatically. Model initialization fails if the
-pipeline does not implement step execution.
-
-Chunk-capable pipelines emit intermediate and final outputs through the same
-request stream. Final-only step pipelines emit only the final result even when
-streaming output is enabled.
-
-The normal non-streaming serving path uses the same internal stream but drains
-it and returns only the final output. Users do not need to enable
-`--diffusion-streaming-output` to benefit from the unified engine lifecycle.
-
-## Send Requests
-
-Execution-mode flags configure the server. They do not change the
-OpenAI-compatible client request format. For example, after starting any of
-the non-streaming configurations above:
-
-```bash
-curl -X POST http://localhost:8091/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "prompt": "a cat sitting on a windowsill",
-    "size": "1024x1024",
-    "num_inference_steps": 50,
-    "seed": 42
-  }' | jq -r '.data[0].b64_json' | base64 -d > cat.png
-```
-
-To create a request-level batch, send multiple independent requests
-concurrently. Do not put several prompts into one `prompt` field:
-
-```bash
-for prompt in "a red fox" "a blue bird" "a green frog"; do
-  curl -s -X POST http://localhost:8091/v1/images/generations \
-    -H "Content-Type: application/json" \
-    -d "{\"prompt\":\"${prompt}\",\"size\":\"1024x1024\",\"seed\":42}" \
-    > /dev/null &
-done
-wait
-```
-
-The scheduler may batch these requests when their sampling parameters are
-compatible. See the
-[Image Generation API](../../serving/image_generation_api.md) for response
-formats and additional client examples. Streaming models can use
-model-specific streaming endpoints documented by their serving guide.
-
-## Python API
-
-The Python arguments mirror the CLI flags:
-
-```python
-from vllm_omni import Omni
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    max_num_seqs=4,
-    request_batch_max_wait_ms=20.0,
-)
-
-outputs = omni.generate(
-    [
-        "a cup of coffee on a table",
-        "a toy dinosaur on a sandy beach",
-        "a fox waking up in bed and yawning",
-    ]
-)
-```
-
-Each list item passed to `Omni.generate()` becomes an independent logical
-request that the scheduler may batch with compatible items.
-
-For step execution:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    step_execution=True,
-    max_num_seqs=1,
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-## Stage Configuration
-
-Use the equivalent engine arguments in a deployment YAML:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    stage_type: diffusion
-    engine_args:
-      step_execution: false
-      max_num_seqs: 4
-      request_batch_max_wait_ms: 20
-```
-
-For step execution, set `step_execution: true` and remove
-`request_batch_max_wait_ms`.
-
-## CLI Reference
-
-| Flag | Default | Effect |
-|---|---:|---|
-| `--step-execution` | disabled | Select step-wise scheduling |
-| `--max-num-seqs` | `1` for diffusion stages | Set request- or step-scheduler capacity |
-| `--request-batch-max-wait-ms` | `0` | Wait for burst coalescing in request mode |
-| `--diffusion-streaming-output` | disabled | Expose supported intermediate diffusion outputs and require step execution |
-
-## Limitations and Troubleshooting
-
-- Requests batch only when their compatibility-sensitive parameters match.
-- Different LoRA adapters or scales run in separate batches.
-- FIFO scheduling can cause an incompatible request to block later compatible
-  requests.
-- All diffusion cache backends are unsupported in step mode. KV transfer is
-  supported for newly admitted step requests; some other request-mode extras
-  remain unsupported.
-- If request-mode startup reports that the pipeline does not support batching,
-  use `--max-num-seqs 1`.
-- If step-mode startup mentions `prepare_encode()`, `denoise_step()`,
-  `step_scheduler()`, or `post_decode()`, the pipeline does not implement the
-  required step contract.
-
-For implementation details and model-author guidance, see
-[Diffusion Continuous Batching](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion/request_batching.md

@@ -0,0 +1,119 @@
+# Request-Level Batching
+
+Request-level batching lets diffusion serving combine multiple compatible
+logical requests into one pipeline `forward()` call. Each prompt remains a
+separate request with its own `request_id`, sampling parameters, seed, output,
+error, and abort state. The scheduler decides which requests can run together.
+
+!!! warning "Prompt List Semantics"
+
+    Diffusion request-level batching does not support a top-level packed
+    list-prompt request. Submit multiple prompts as independent requests and let
+    the scheduler batch compatible in-flight requests. Multimodal payloads stay
+    inside a single prompt dict, for example
+    `{"prompt": "...", "multi_modal_data": {"image": image}}`.
+
+## Enablement
+
+Request-level execution is the default diffusion request mode when
+`step_execution=False`. `max_num_seqs` controls the maximum number of compatible
+requests in one scheduler wave:
+
+- `max_num_seqs=1` is the serial request path.
+- `max_num_seqs>1` allows fused request-level batching for pipelines that
+  declare request-batch support.
+
+Increase `max_num_seqs` above `1` to allow the request scheduler to keep more
+than one compatible request active:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4
+```
+
+For bursty online traffic, you can also set a small admission wait window. This
+lets the engine wait briefly before the first `schedule()` of a new wave so
+nearby compatible requests can arrive and share the same fused forward pass:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+`--request-batch-max-wait-ms 0` is the default and disables admission waiting,
+so there is no added wait latency.
+
+For deploy YAMLs, configure the diffusion stage engine args:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    stage_type: diffusion
+    engine_args:
+      max_num_seqs: 4
+      request_batch_max_wait_ms: 20
+```
+
+## Compatibility
+
+Only pipelines that declare request-batch support can use the fused
+multi-request `pipeline.forward(batch)` path. Pipelines that do not support this
+contract can still use request-level execution with `max_num_seqs=1`, which is
+the serial request path. Setting `max_num_seqs>1` for a pipeline without
+request-batch support fails early during engine initialization.
+
+The scheduler batches only compatible requests. Compatibility is based on
+shape-sensitive and guidance-sensitive sampling fields, including resolution,
+frame count, CFG settings, output count, and LoRA identity. Requests with
+different LoRA adapters or scales are kept in separate batches so the worker
+activates one adapter per batch.
+
+Request-level batching applies when `step_execution=False`. For the separate
+step-wise runtime, see [Step Execution](step_execution.md).
+
+## Tuning
+
+- `max_num_seqs` caps the number of active compatible requests in one scheduler
+  wave. `1` means serial request execution; values above `1` enable fused
+  request-level batching when the pipeline supports it.
+- `request_batch_max_wait_ms` is an upper bound on extra admission wait before a
+  new wave starts. Keep it small for latency-sensitive serving; values such as
+  `10` to `50` ms are a practical starting range for bursty HTTP ingress.
+- `0` disables admission waiting and preserves the lowest first-request latency.
+- FIFO ordering is conservative: an incompatible request at the front of the
+  waiting queue can block later compatible requests from joining the current
+  batch.
+
+## Python API
+
+When constructing `Omni`, pass the same engine arguments:
+
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    max_num_seqs=4,
+    request_batch_max_wait_ms=20.0,
+)
+
+outputs = omni.generate(
+    [
+        "a cup of coffee on a table",
+        "a toy dinosaur on a sandy beach",
+        "a fox waking up in bed and yawning",
+    ]
+)
+```
+
+`Omni.generate([...])` submits each list item as its own logical diffusion
+request. The runtime may batch those requests internally when their sampling
+parameters are compatible.
+
+## For Contributors
+
+For implementation details and model-author guidance, see
+[Request-Level Batching for Diffusion](../../design/feature/diffusion_request_level_batching.md).

docs/user_guide/diffusion/step_execution.md

@@ -0,0 +1,100 @@
+# Step Execution
+
+Step execution is an opt-in diffusion execution mode enabled with
+`step_execution=True` when constructing `Omni`.
+
+It is not a generic diffusion toggle for every pipeline. Only pipelines that
+implement the stepwise contract support it today.
+
+## Quick Start
+
+### Python API
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    step_execution=True,
+)
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(
+        num_inference_steps=50,
+    ),
+)
+```
+
+### Serving
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+For serving, `--step-execution` enables the step-wise runtime. Continuous
+batching only becomes relevant when `--max-num-seqs > 1`.
+
+`max_num_seqs` controls the step scheduler capacity:
+
+- `max_num_seqs=1` runs one request through the step-wise state machine.
+- `max_num_seqs>1` allows compatible active requests to share step waves.
+
+## Supported Pipelines
+
+| Pipeline | Example models | Step execution |
+|----------|----------------|----------------|
+| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
+| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
+
+!!! warning "Experimental continuous batching"
+    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
+    the step-wise path can batch
+    compatible requests together. This is experimental. Requests with
+    incompatible sampling parameters are intentionally kept in separate batches,
+    and `max_num_seqs=1` remains the conservative single-request step path.
+
+## Current Limitations
+
+- Continuous batching under `step_execution` is experimental and only batches
+  compatible requests.
+- `cache_backend` is not supported together with step execution.
+- Unsupported pipelines fail early during model loading.
+- Request-mode extras such as KV transfer are not wired into step mode yet.
+- LoRA is supported in step mode, but each batch must use a single adapter:
+  requests with different `lora_request` or `lora_scale` are scheduled into
+  separate batches.
+
+## When To Use It
+
+Use step execution only when you specifically need the pipeline to run through
+its stepwise request state machine. For normal diffusion inference, leave it
+disabled unless your workflow depends on this mode.
+
+For Qwen-Image online serving, the usual progression is:
+
+- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
+- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
+
+If you are looking for general diffusion speedups, see
+[Diffusion Features Overview](../diffusion_features.md).
+
+## Troubleshooting
+
+If model loading fails with a message mentioning `prepare_encode()`,
+`denoise_step()`, `step_scheduler()`, and `post_decode()`, the selected
+pipeline does not support step execution.
+
+## For Model Authors
+
+If you want to add step execution support to a new diffusion pipeline, see the
+implementation guide:
+[Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
+
+If you also want that pipeline to participate in the experimental batched
+step-wise path, see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion_features.md

@@ -59,7 +59,7 @@ Memory optimization methods help reduce GPU memory usage, enabling inference on
 
 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[CPU Offload](diffusion/cpu_offload.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
+| **[CPU Offload](diffusion/cpu_offload_diffusion.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
 | **[Quantization](quantization/overview.md)** | Reduces transformer stages from BF16 to FP8/INT8/etc. | Limited VRAM, minimal accuracy loss    |
 | **[VAE Parallelism](diffusion/parallelism/vae_parallelism.md)** | Distributes VAE decode work across GPUs | High-resolution generation with reduced VAE memory peak |
 
@@ -80,11 +80,12 @@ denoise steps.
 
 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[Diffusion Execution Modes](diffusion/execution_modes.md)** | Configures serial requests, request batching, step execution, continuous batching, and streaming output | Matching latency, throughput, cancellation, and output-delivery requirements |
+| **[Request-Level Batching](diffusion/request_batching.md)** | Scheduler batches compatible independent diffusion requests into one pipeline forward pass | Bursty online serving and multi-request throughput |
+| **[Step Execution](diffusion/step_execution.md)** | Per-step denoise execution with mid-request abort support | Request cancellation between denoise steps, fine-grained execution control |
 
-**Note:** Request-level batching and step execution are capability-based.
+**Note:** Request-level batching is available for pipelines that declare the
-Consult the execution guide and the selected pipeline's documentation for
+request-batch forward contract. Step execution is currently supported by
-current support.
+QwenImagePipeline only. See [Supported Models](#supported-models) for details.
 
 ### Quantization Methods
 
@@ -104,8 +105,8 @@ The following tables show which models support each feature:
 
 > Notes:
 
-> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
+> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload_diffusion.md).
-> 2. The **💾Quantization** column is collapsed for readability. See [Quantization](quantization/overview.md) for per-method and per-model support details.
+> 2. The **💾Quantization** column is collapsed for readability. See [Quantization Overview](quantization/overview.md) for per-method and per-model support details.
 
 ### ImageGen
 
@@ -159,7 +160,7 @@ The following tables show which models support each feature:
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
-| **MiniMax-H3**               |     ❌     |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
+
 
 **Frame Interpolation Support**
 
@@ -205,7 +206,7 @@ The following tables show which models support each feature:
     3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
     4. CPU Offloading (Layerwise) supports single-card for now.
     5. Using FP8-Quant as an example of qunatization methods.
-    6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
+    6. Step Execution is not compatible with cache backends (TeaCache, Cache-DiT). LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
 
 
 ## Multi-Thread Weight Loading
@@ -274,9 +275,9 @@ Measured on NVIDIA H800:
 
 **Memory Optimization:**
 
-- **[CPU Offload Guide](diffusion/cpu_offload.md)** - Offload model components to CPU, reduce GPU memory usage
+- **[CPU Offload Guide](diffusion/cpu_offload_diffusion.md)** - Offload model components to CPU, reduce GPU memory usage
 - **[VAE Parallelism Guide](diffusion/parallelism/vae_parallelism.md)** - Distribute VAE decode work across GPUs for high-resolution images and videos
-- **[Quantization](quantization/overview.md)** - Quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
+- **[Quantization Overview](quantization/overview.md)** - Overview of quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
 
 **Extensions:**
 
@@ -285,7 +286,7 @@ Measured on NVIDIA H800:
 
 **Execution Modes:**
 
-- **[Diffusion Execution Modes](diffusion/execution_modes.md)** - Configure request batching, step execution, continuous batching, and streaming output
+- **[Step Execution Guide](diffusion/step_execution.md)** - Per-step denoise execution with mid-request abort support
 
 **Startup Optimization:**
 

docs/user_guide/examples/offline_inference/text_to_image.md

@@ -184,7 +184,7 @@ if __name__ == "__main__":
 !!! info
 
     For diffusion request-level batching controls such as `max_num_seqs`, see
-    [Diffusion Execution Modes](../../diffusion/execution_modes.md).
+    [Request-Level Batching](../../diffusion/request_batching.md).
 
 ### Negative Prompts
 

docs/user_guide/examples/online_serving/bagel.md

@@ -40,7 +40,7 @@ bash run_server_stage_cli.sh --stage 0
 bash run_server_stage_cli.sh --stage 1
 ```
 
-To use a custom deploy YAML:
+To use a custom deploy YAML (note: `--stage-configs-path` is deprecated in favor of `--deploy-config`):
 
 ```bash
 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \

docs/user_guide/examples/online_serving/glm_image.md

@@ -11,8 +11,8 @@ memory, sampling params) live in `vllm_omni/deploy/glm_image.yaml`.
 vllm serve zai-org/GLM-Image --omni --port 8091
 ```
 
-The config system auto-detects the pipeline from the model's `model_index.json`, so
+The config system auto-detects the pipeline from the model's `model_index.json` — no
-no manual `--deploy-config` is needed.
+manual `--stage-configs-path` or `--deploy-config` needed.
 
 By default, stage 0 (AR) runs on GPU 0 and stage 1 (Diffusion) on GPU 1. To colocate
 both stages on a single GPU, override per stage:

docs/user_guide/examples/online_serving/qwen2_5_omni.md

@@ -15,9 +15,9 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-To use a custom deploy config, launch the server with the command below:
+If you have custom stage configs file, launch the server with command below
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```
 
 ### Send Multi-modal Request
@@ -182,7 +182,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
-- `--deploy-config`: Path to a custom deploy config YAML file (optional)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -195,9 +195,9 @@ The script supports the following arguments:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
 
-To use a custom deploy config:
+If you have custom stage configs file:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```
 
 **Step 2: Run the Gradio demo**

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -306,7 +306,7 @@ The gradio script supports the following arguments:
     ``````
 ??? abstract "qwen3_omni_moe_thinking.yaml"
     ``````yaml
-    --8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml"
+    --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
     ``````
 ??? abstract "run_curl_multimodal_generation.sh"
     ``````sh

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -422,19 +422,12 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
 
 ### Streaming WebSocket
-The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```
 
-`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
-```bash
-python qwen3_tts/streaming_speech_client.py \
-    --text "First utterance." \
-    --text "Second utterance, same connection."
-```
-
 ### Gradio demos
 ```bash
 ./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)

docs/user_guide/fault_injection_reliability_matrix.md

@@ -69,19 +69,19 @@ The behaviors and conclusions above are summarized from current fault injection
 
 ### Example: `SIGTERM` fault injection log (Qwen3-Omni)
 
-The following log is based on a real run where a `SIGTERM` signal was sent to the serve root process. Its command line has been normalized to the current `--deploy-config` option.
+The following full log is from a real run where a `SIGTERM` signal was sent to the serve root process.
 
 ```text
-Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 Server ready on 127.0.0.1:60675 (OmniServer startup took 206.023s)
 OmniServer started successfully
-[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [reliability][process-kill] current_server_proc pid=557118 name=python3 cmdline=/workspace/.venv/bin/python3 -c from multiprocessing.resource_tracker import main;main(57)
 [reliability][process-kill] current_server_proc pid=557119 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=557122 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=558201 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 
-[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [omni_base.py:463] [AsyncOmni] Shutting down
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [async_omni_engine.py:2397] [AsyncOmniEngine] Shutting down Orchestrator
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [orchestrator.py:351] [Orchestrator] Received shutdown signal