Revert "[BugFix] Fix 'GPUARWorker' attribute error (#5478 )"

This reverts commit f4640d9a2e.
2026-07-28 21:00:36 +08:00
593 changed files with 10507 additions and 48075 deletions
@@ -89,7 +89,7 @@ steps:
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
-        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+        - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py

    - label: "Diffusion Model Test"
      agent_pool: mi325_1
@@ -35,7 +35,7 @@ steps:
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
-    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"


 - label: "Voxtral TTS CUDA Unit Test"
@@ -8,24 +8,14 @@ steps:
  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-merge-pipeline
    steps:
-      - label: "Simple · Model Executor Test"
+      - label: "Simple · Diffusion & Model Executor Test"
        commands:
-          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Diffusion Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Engine&Entrypoints Test"
-        commands:
-          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        mirror_hardwares: l4_1

      - label: "Simple · Other Test"
        commands:
-          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: Diffusion Test"
@@ -37,6 +27,12 @@ steps:
          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
        mirror_hardwares: l4_1

+      - label: "Diffusion · Offloader Test"
+        timeout_in_minutes: 40
+        commands:
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
+        mirror_hardwares: l4_1
+
  - label: "Distributed Test"
    depends_on: upload-merge-pipeline
    timeout_in_minutes: 60
@@ -139,32 +135,39 @@ steps:
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        mirror_hardwares: h100_2
-
-      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - tests/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/entrypoints/cli/serve.py
-        timeout_in_minutes: 50
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        mirror_hardwares: h100_2
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate

      - label: "TTS · VoxCPM2 Test"
        source_file_dependencies:
@@ -182,6 +185,22 @@ steps:
            "
        mirror_hardwares: l4_1

+      - label: "TTS · SoulX-Singer Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_soulxsinger.py
+          - vllm_omni/diffusion/models/soulx_singer/
+          - vllm_omni/deploy/soulxsinger_svc.yaml
+          - vllm_omni/deploy/soulxsinger_svs.yaml
+        timeout_in_minutes: 30
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
      - label: "Diffusion · Qwen Image Edit Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_qwen_image_edit.py
@@ -15,13 +15,7 @@ steps:
      - label: ":full_moon: Omni · Function Test with H100"
        timeout_in_minutes: 120
        commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-        mirror_hardwares: h100_2
-
-      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
-        timeout_in_minutes: 50
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
        mirror_hardwares: h100_2

      - label: ":full_moon: Omni · Doc Test with L4"
@@ -427,12 +421,6 @@ steps:
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
        mirror_hardwares: h100_2

-      - label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test"
-        timeout_in_minutes: 180
-        commands:
-          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py -m full_model --run-level full_model
-        mirror_hardwares: h100_4
-
      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
        timeout_in_minutes: 180
        commands:
@@ -461,47 +449,36 @@ steps:
        mirror_hardwares: h100_2

  - group: ":card_index_dividers: Diffusion Test"
-    key: nightly-diffusion-test-group
    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
-      - label: ":full_moon: Diffusion · Distributed Test with L4"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
-        mirror_hardwares: l4_4
-
-      - label: ":full_moon: Diffusion · Distributed Test with H100"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_4

      - label: ":full_moon: Diffusion · Offloader Test"
-        timeout_in_minutes: 60
+        timeout_in_minutes: 40
        commands:
-          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda"
        mirror_hardwares: l4_1

-      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
-        timeout_in_minutes: 60
+      - label: ":full_moon: Diffusion · Quantization Test with H100"
        commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+          - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
        mirror_hardwares: h100_1

-      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
-        mirror_hardwares: h100_2
-
      - label: ":full_moon: Diffusion · Quantization Test with L4"
        timeout_in_minutes: 60
        commands:
-          - pip install "vllm-gguf-plugin>=0.0.3"
          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
        mirror_hardwares: l4_1

+      - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
  - label: ":bar_chart: Testcase Statistics"
    key: nightly-testcase-statistics
    timeout_in_minutes: 120
@@ -8,26 +8,22 @@ steps:
  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-ready-pipeline
    steps:
-      - label: "Simple · Model Executor Test"
+      - label: "Simple · Diffusion & Model Executor Test"
        commands:
-          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Diffusion Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        mirror_hardwares: l4_1
-
-      - label: "Simple · Engine&Entrypoints Test"
-        commands:
-          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        mirror_hardwares: l4_1

      - label: "Simple · Other Test"
        commands:
-          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        mirror_hardwares: l4_1

+  - label: "Custom Pipeline Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
+    mirror_hardwares: l4_1
+
  - group: ":card_index_dividers: Diffusion Test"
    depends_on: upload-ready-pipeline
    steps:
@@ -108,34 +104,11 @@ steps:
  - group: ":card_index_dividers: E2E Test"
    depends_on: upload-ready-pipeline
    steps:
-      - label: "Custom Pipeline Test"
-        source_file_dependencies:
-          - tests/e2e/features/custom_pipeline/
-          - tests/e2e/features/helpers/custom_pipeline.py
-          - vllm_omni/diffusion/
-          - vllm_omni/engine/
-          - vllm_omni/entrypoints/
-          - vllm_omni/inputs/
-          - vllm_omni/outputs/
-        commands:
-          - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "RLHF · VeRL-Omni E2E Test"
        source_file_dependencies:
-          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
-          - tests/e2e/features/helpers/verl_omni_server.py
-          - tests/e2e/features/helpers/custom_pipeline.py
+          - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/offline_inference/custom_pipeline/
          - vllm_omni/diffusion/
          - vllm_omni/engine/
          - vllm_omni/entrypoints/
@@ -144,7 +117,7 @@ steps:
        commands:
          - |
            timeout 20m bash -c "
-              pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+              pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
            "
        mirror_hardwares: l4_1

@@ -170,7 +143,34 @@ steps:
            timeout 20m bash -c "
              pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
            "
-        mirror_hardwares: h100_2
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate

      - label: "Omni · MiniCPM-o 4.5 Online Test"
        source_file_dependencies:
@@ -181,34 +181,34 @@ steps:
            timeout 20m bash -c "
              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
            "
-        mirror_hardwares: h100_2
-
-      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
-        source_file_dependencies:
-          - tests/assets/minicpmo_4_5/response_required_16k.wav
-          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
-          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
-          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
-          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
-          - tests/helpers/minicpmo_4_5_duplex.py
-          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
-          - vllm_omni/experimental/fullduplex/
-          - vllm_omni/model_executor/models/minicpmo_4_5/
-          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
-          - vllm_omni/core/sched/
-          - vllm_omni/outputs/multimodal_accumulation.py
-          - vllm_omni/engine/stage_init_utils.py
-          - vllm_omni/engine/async_omni_engine.py
-          - vllm_omni/entrypoints/cli/serve.py
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
-            "
-        mirror_hardwares: h100_2
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate

      - label: "TTS · VoxCPM2 Test"
        source_file_dependencies:
@@ -344,3 +344,13 @@ steps:
              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
            "
        mirror_hardwares: h100_1
+
+      - label: "Diffusion · GGUF Plugin Test"
+        soft_fail: true
+        source_file_dependencies:
+          - tests/plugin/test_gguf_diffusion.py
+          - vllm_omni/diffusion/model_loader
+        commands:
+          - pip install "vllm-gguf-plugin>=0.0.3"
+          - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
+        mirror_hardwares: l4_1
@@ -10,7 +10,7 @@ steps:
          DOCKER_BUILDKIT: "1"
          # Buildkite will automatically replace this with the actual commit hash
          VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.26.0"
+          VLLM_VERSION: "v0.25.0"
        priority: 100
        timeout_in_minutes: 60
        soft_fail: false
@@ -95,80 +95,6 @@ steps:
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
-  - group: ":card_index_dividers: Diffusion I2I Model Test"
-    key: nightly-HunyuanImage3-x2i-test-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
-      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
-          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
-  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
-    key: nightly-diffusion-benchmark-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
-        timeout_in_minutes: 180
-        mirror_hardwares: a3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
  - group: ":card_index_dividers: Diffusion X2V Model Test"
    key: nightly-diffusion-x2v-group
    depends_on: upload-nightly-pipeline
@@ -177,14 +103,6 @@ steps:
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2v-test"
    steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
-        mirror_hardwares: a2b3_npu_4
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
      - label: ":full_moon: Diffusion X2V · Perf Test"
        key: nightly-diffusion-x2v-performance
        timeout_in_minutes: 180
@@ -16,15 +16,15 @@
 /.claude/ @hsliuustc0106

 # Tests
-/tests/ @yenuo26 @NickCao
+/tests/ @yenuo26

 # CI
-/.buildkite/ @yenuo26 @congw729 @NickCao
+/.buildkite/ @yenuo26 @congw729

 # Engine and model_executor
 /vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307
 /vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan @Sy0307
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks @NickCao
+/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks
 /vllm_omni/model_executor/ @tzhouam @gcanlin
 /vllm_omni/worker/ @tzhouam @gcanlin

@@ -37,7 +37,7 @@

 # Omni and TTS support
 /vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin @NickCao
+/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin

 # Benchmarks
 /benchmarks/accuracy/ @david6666666
@@ -16,10 +16,14 @@ Easy, fast, and cheap omni-modality model serving for everyone
 ---

 *Latest News* 🔥
- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
- [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.
+- [2026/06] We released [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) - an **omnimodal world-model** release aligned with vLLM 0.22, featuring [Nvidia Cosmos3](recipes/cosmos3/Cosmos3-Nano.md)/DreamZero world model support, expanded quantization coverage across Blackwell/NPU/XPU, TTS production improvements, new models including MiniCPM-o 4.5, MOSS-TTS, and Lance, plus RL integration with [VeRL-Omni](https://github.com/verl-project/verl-omni).
+- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
+- [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
+- [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
+- [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
+- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.

 ---
@@ -51,13 +55,12 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
- Full-duplex realtime serving with streaming audio input and output (experimental)

 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
+- **Omni-modality models** (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
 - **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
+- **Diffusion models** — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
 - **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)

 ## Getting Started
@@ -47,7 +47,7 @@ If no, check your shell running the ComfyUI process. There may be some error mes
 This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):

 - **Generate Image** for text-to-image and image-to-image tasks
- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
+- **Generate Video** for text-to-video and image-to-video tasks
 - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
 - **TTS** and **TTS Voice Clone** for TTS tasks

@@ -105,7 +105,7 @@ You can configure per-stage sampling parameters for multi-stage models.
 > [!TIP]
 > The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.

-### Text-to-video and image-to-video generation (e.g., Wan, MiniMax-H3)
+### Text-to-video and image-to-video generation (e.g., Wan)

 (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Video Generation**)

@@ -117,11 +117,7 @@ You can configure per-stage sampling parameters for multi-stage models.
 </p>

 > [!TIP]
-> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
->
-> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
->
-> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
+> The node automatically choose text-to-video or image-to-video API endpoints depending on whether you connect an image input or not.

 ### TTS (e.g., Qwen TTS series)

@@ -15,13 +15,11 @@ from .comfyui_vllm_omni.nodes import (
    VLLMOmniDiffusionSampling,
    VLLMOmniGenerateImage,
    VLLMOmniGenerateVideo,
-    VLLMOmniMiniMaxH3Params,
    VLLMOmniQwenTTSParams,
    VLLMOmniRemoteLoRA,
    VLLMOmniSamplingParamsList,
    VLLMOmniTTS,
    VLLMOmniUnderstanding,
-    VLLMOmniVideoReferences,
    VLLMOmniVoiceClone,
    VLLMOmniWanParams,
 )
@@ -34,7 +32,6 @@ NODE_CLASS_MAPPINGS = {
    "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
    "VLLMOmniTTS": VLLMOmniTTS,
    "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
-    "VLLMOmniVideoReferences": VLLMOmniVideoReferences,
    # === Params ===
    "VLLMOmniARSampling": VLLMOmniARSampling,
    "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -42,7 +39,6 @@ NODE_CLASS_MAPPINGS = {
    "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
    "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
    "VLLMOmniWanParams": VLLMOmniWanParams,
-    "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
 }

 # A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -53,7 +49,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
    "VLLMOmniUnderstanding": "Multimodality Understanding",
    "VLLMOmniTTS": "TTS (Text to Speech)",
    "VLLMOmniVoiceClone": "TTS Voice Cloning",
-    "VLLMOmniVideoReferences": "Video References",
    # === Params ===
    "VLLMOmniARSampling": "AR Sampling Params",
    "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
@@ -61,7 +56,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
    "VLLMOmniRemoteLoRA": "LoRA",
    "VLLMOmniQwenTTSParams": "Qwen TTS Params",
    "VLLMOmniWanParams": "Wan Video Params",
-    "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",
 }

 WEB_DIRECTORY = "./web"
@@ -10,9 +10,7 @@ from .utils.types import (
    AudioFormat,
    AutoregressionSamplingParams,
    DiffusionSamplingParams,
-    MiniMaxH3ModelSpecificParams,
    QwenTTSModelSpecificParams,
-    VideoReferences,
    WanModelSpecificParams,
 )
 from .utils.validators import (
@@ -165,8 +163,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
                "num_frames": ("INT", {"default": 41, "min": 1}),
            },
            "optional": {
-                "frame": ("IMAGE",),
-                "references": ("VIDEO_REFERENCES",),
+                "image": ("IMAGE",),
                "sampling_params": ("SAMPLING_PARAMS",),
                "lora": ("REMOTE_LORA",),
                "model_params": ("VIDEO_PARAMS",),
@@ -177,15 +174,6 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
    RETURN_NAMES = ("video",)
    FUNCTION = "generate"

-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
-        base = super().VALIDATE_INPUTS(url, model)
-        if base is not True:
-            return base
-        if frame is not None and references is not None:
-            return "Provide only one of frame or references, not both."
-        return True
-
    async def generate(
        self,
        url: str,
@@ -196,8 +184,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
        fps: int,
        num_frames: int,
        negative_prompt: str | None = None,
-        frame: torch.Tensor | None = None,
-        references: dict | None = None,
+        image: torch.Tensor | None = None,
        sampling_params: dict | list[dict] | None = None,
        model_params: dict | None = None,
        lora: dict | None = None,
@@ -220,16 +207,14 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):

        if sampling_params is not None:
            sampling_params.pop("type", None)  # internal fields
-        # model_params["type"] is also an internal field, but is used in api_client
-        # if model_params is not None:
-        #     model_params.pop("type", None)  # internal fields
+        if model_params is not None:
+            model_params.pop("type", None)  # internal fields

        client = VLLMOmniClient(url)
        output = await client.generate_video(
            model=model,
            prompt=prompt,
-            frame=frame,  # frame present => fl2va / Wan I2V
-            references=references,
+            image=image,  # image present => i2v, absent => t2v
            width=width,
            height=height,
            num_frames=num_frames,
@@ -749,78 +734,3 @@ class VLLMOmniWanParams:

    def get_params(self, **kwargs):
        return (WanModelSpecificParams(kwargs),)
-
-
-class VLLMOmniMiniMaxH3Params:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "audio_flow_shift": (
-                    "FLOAT",
-                    {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1},
-                ),
-                "flow_shift": (
-                    "FLOAT",
-                    {"default": 12.0, "min": 0.0, "max": 100.0, "step": 0.1},
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("VIDEO_PARAMS",)
-    RETURN_NAMES = ("MiniMax-H3 video params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Video Params"
-
-    def get_params(self, **kwargs):
-        params = MiniMaxH3ModelSpecificParams(kwargs)
-        params["type"] = "minimax_h3"
-        return (params,)
-
-
-class VLLMOmniVideoReferences:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {},
-            "optional": {
-                "image_1": ("IMAGE",),
-                "image_2": ("IMAGE",),
-                "audio_1": ("AUDIO",),
-                "audio_2": ("AUDIO",),
-                "video_1": ("VIDEO",),
-                "video_2": ("VIDEO",),
-            },
-        }
-
-    RETURN_TYPES = ("VIDEO_REFERENCES",)
-    RETURN_NAMES = ("references",)
-    FUNCTION = "get_references"
-    CATEGORY = "vLLM-Omni"
-
-    def get_references(
-        self,
-        image_1: torch.Tensor | None = None,
-        image_2: torch.Tensor | None = None,
-        audio_1: AudioInput | None = None,
-        audio_2: AudioInput | None = None,
-        video_1: VideoInput | None = None,
-        video_2: VideoInput | None = None,
-        **kwargs,
-    ):
-        if kwargs:
-            logger.info("Uncaught kwargs: %s", kwargs)
-        refs = VideoReferences()
-        if image_1 is not None:
-            refs["image_1"] = image_1
-        if image_2 is not None:
-            refs["image_2"] = image_2
-        if audio_1 is not None:
-            refs["audio_1"] = audio_1
-        if audio_2 is not None:
-            refs["audio_2"] = audio_2
-        if video_1 is not None:
-            refs["video_1"] = video_1
-        if video_2 is not None:
-            refs["video_2"] = video_2
-        return (refs,)
@@ -24,7 +24,6 @@ from .format import (
    image_tensor_to_base64,
    image_tensor_to_png_bytes,
    video_to_base64,
-    video_to_bytes,
 )
 from .logger import get_logger, pretty_printer
 from .models import lookup_model_spec
@@ -64,11 +63,7 @@ async def url_bytes(session: aiohttp.ClientSession, url: str, verb: str = "get",

 class VLLMOmniClient:
    def __init__(
-        self,
-        base_url: str,
-        timeout: float | None = None,
-        poll_interval: float = 5.0,
-        max_poll_duration: float = 60 * 30,
+        self, base_url: str, timeout: float | None = None, poll_interval: float = 5.0, max_poll_duration: float = 60 * 5
    ):
        self.base_url = base_url
        self.timeout = aiohttp.ClientTimeout(total=timeout)
@@ -266,17 +261,12 @@ class VLLMOmniClient:
        num_frames: int,
        fps: int,
        negative_prompt: str | None = None,
-        frame: torch.Tensor | None = None,
-        references: dict | None = None,
+        image: torch.Tensor | None = None,
        sampling_params: dict | None = None,
        model_params: dict | None = None,
        lora: dict | None = None,
-        **extra_params,
+        **extra_body,
    ) -> VideoInput:
-        if frame is not None and references is not None:
-            raise ValueError("Provide only one of frame or references, not both.")
-
-        # === regular payload fields ===
        form = aiohttp.FormData()
        form.add_field("model", model)
        form.add_field("prompt", prompt)
@@ -289,91 +279,23 @@ class VLLMOmniClient:
        if sampling_params is not None:
            for k, v in sampling_params.items():
                form.add_field(k, str(v))
+        if model_params is not None:
+            for k, v in model_params.items():
+                form.add_field(k, str(v))
        if lora is not None:
            form.add_field("lora", json.dumps(lora, ensure_ascii=False))
+        if extra_body:
+            form.add_field("extra_body", json.dumps(extra_body, ensure_ascii=False))

-        # === multimodal inputs (first-last-frames, references, etc.) ===
-        input_reference_image: torch.Tensor | None = None
-        audio_reference: AudioInput | None = None
-        reference_videos: list[VideoInput] = []
-        video_task: str | None = None
-
-        if frame is not None:
-            input_reference_image = frame
-            video_task = "fl2va"
-        elif references is not None:
-            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
-            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
-            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
-
-            if not images and not audios and not videos:
-                raise ValueError("references is empty; connect at least one image/audio/video.")
-
-            if videos:
-                if images or audios:
-                    raise ValueError(
-                        "Video references cannot be combined with image or audio references. "
-                        "Connect only video_1/video_2 for multi-video Ref2VA."
-                    )
-                reference_videos = videos
-                video_task = "ref2va"
-            elif len(images) == 1 and len(audios) == 1:
-                input_reference_image = images[0]
-                audio_reference = audios[0]
-                video_task = "ref2va"
-            else:
-                raise ValueError(
-                    "Invalid references combination. Supported modes: "
-                    "(1) one or more videos only, or (2) exactly one image and one audio."
-                )
-        else:
-            video_task = "t2va"
-
-        if input_reference_image is not None:
+        if image is not None:
            image_filename = "image.png"  # Required for multipart form
            form.add_field(
                "input_reference",
-                image_tensor_to_png_bytes(input_reference_image, image_filename),
+                image_tensor_to_png_bytes(image, image_filename),
                filename=image_filename,
                content_type="image/png",
            )

-        if audio_reference is not None:
-            form.add_field(
-                "audio_reference",
-                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
-            )
-
-        for idx, video in enumerate(reference_videos, start=1):
-            video_filename = f"reference_{idx}.mp4"
-            form.add_field(
-                "input_references",
-                video_to_bytes(video, video_filename),
-                filename=video_filename,
-                content_type="video/mp4",
-            )
-
-        # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
-        if model_params is not None:
-            model_params = dict(model_params)
-            model_params.pop("type", None)
-
-        spec, _ = lookup_model_spec(model)
-        params_builder = spec.get("params_builder") if spec else None
-        if params_builder is not None:
-            form_fields = params_builder(
-                model_params or {},
-                extra_params={**extra_params, "task": video_task},
-            )
-            for k, v in form_fields.items():
-                form.add_field(k, v if isinstance(v, str) else str(v))
-        else:
-            if model_params is not None:
-                for k, v in model_params.items():
-                    form.add_field(k, str(v))
-            if extra_params:
-                form.add_field("extra_params", json.dumps(extra_params, ensure_ascii=False))
-
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            # Start the video generation job
            url = f"{self.base_url}/videos"
@@ -1,12 +1,7 @@
-import json
 import re
-from typing import Any

-from .logger import get_logger
 from .types import Modality, ModelMode, Spec

-logger = get_logger(__name__)
-

 def _bagel_payload_preprocessor(payload: dict) -> dict:
    try:
@@ -34,30 +29,6 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
    return payload


-def _minimaxh3_params_builder(
-    model_params: dict[str, Any],
-    *,
-    extra_params: dict[str, Any],
-) -> dict[str, Any]:
-    """Build multipart form fields for MiniMax-H3 from model_params + routed task."""
-    params = dict(model_params)
-    params.pop("type", None)
-    form_fields: dict[str, Any] = {}
-    merged_extra_params: dict[str, Any] = dict(extra_params or {})
-
-    if "flow_shift" in params:
-        form_fields["flow_shift"] = params.pop("flow_shift")
-    for key in ("audio_flow_shift",):
-        if key in params:
-            merged_extra_params[key] = params.pop(key)
-    if params:
-        logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
-
-    if merged_extra_params:
-        form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
-    return form_fields
-
-
 _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
    r"BAGEL-7B-MoT": {
        "stages": [
@@ -100,21 +71,6 @@ _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
            }
        ],
    },
-    r"MiniMax-H3(/FL2VA|/Ref2VA)?": {
-        "stages": ["diffusion"],
-        "modes": [
-            {
-                "mode": ModelMode.VIDEO_GENERATION,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-        "params_builder": _minimaxh3_params_builder,
-    },
 }
 # Convert dict keys to regex patterns
 MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
@@ -124,10 +80,12 @@ del _MODEL_PIPELINE_SPECS


 def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
-    normalized = model.rstrip("/").lstrip("/")
-    lookup_key = normalized.split("/", 1)[-1]
+    try:
+        last_component = model.rstrip("/").rsplit("/", 1)[-1]
+    except IndexError:
+        last_component = model
    for pattern, spec in MODEL_PIPELINE_SPECS.items():
-        if pattern.search(lookup_key):
+        if pattern.search(last_component):
            return spec, pattern.pattern
    return None, None

@@ -143,9 +101,6 @@ if __name__ == "__main__":
        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        "Custom/Path/UnknownModel-Instruct",
        "Not/Matching/Anything",
-        "/path/to/MiniMax-H3/FL2VA",
-        "/path/to/MiniMax-H3/Ref2VA",
-        "MiniMaxAI/MiniMax-H3",
    ]

    test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
@@ -157,13 +112,7 @@ if __name__ == "__main__":
            if preprocessor := spec.get("payload_preprocessor"):
                result = preprocessor(test_payload)
                print(f"✓ {path:<40} → {result}")
-            elif params_builder := spec.get("params_builder"):
-                result = params_builder(
-                    {"audio_flow_shift": 3.0, "flow_shift": 12.0, "type": "minimax_h3"},
-                    extra_params={"task": "t2va"},
-                )
-                print(f"✓ {path:<40} → {result}")
            else:
-                print(f"✓ {path:<40} → No preprocessor/params_builder")
+                print(f"✓ {path:<40} → No preprocessor")
        else:
            print(f"✗ {path:<40} → No match")
@@ -1,8 +1,12 @@
 from collections.abc import Callable
 from enum import Enum, auto
-from typing import Any, Literal, TypeAlias
-
-from typing_extensions import NotRequired, TypedDict
+from typing import (
+    Any,
+    Literal,
+    NotRequired,
+    TypeAlias,
+    TypedDict,
+)

 AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"]

@@ -23,14 +27,6 @@ class WanModelSpecificParams(dict):
    pass


-class MiniMaxH3ModelSpecificParams(dict):
-    pass
-
-
-class VideoReferences(dict):
-    pass
-
-
 class ModelMode(Enum):
    IMAGE_GENERATION = auto()
    VIDEO_GENERATION = auto()
@@ -51,11 +47,9 @@ class ModelModeSpec(TypedDict):


 PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
-ParamsBuilder: TypeAlias = Callable[..., dict[str, Any]]


 class Spec(TypedDict):
    stages: list[Literal["diffusion", "autoregression"]]
    modes: list[ModelModeSpec]
    payload_preprocessor: NotRequired[PayloadPreprocessor]
-    params_builder: NotRequired[ParamsBuilder]
@@ -1 +1,513 @@
-{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated
+{
+    "id": "3aea1d08-7de5-4997-ba71-b2f66ad72a21",
+    "revision": 0,
+    "last_node_id": 14,
+    "last_link_id": 28,
+    "nodes": [
+        {
+            "id": 10,
+            "type": "MarkdownNote",
+            "pos": [
+                227.99306819575278,
+                -231.24143306069843
+            ],
+            "size": [
+                240.3326478922474,
+                136.3505820791881
+            ],
+            "flags": {},
+            "order": 0,
+            "mode": 0,
+            "inputs": [],
+            "outputs": [],
+            "title": "Note: Task and Input",
+            "properties": {},
+            "widgets_values": [
+                "vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both text-to-video generation and image-to-video generation. The node will create corresponding payload depending on whether an input image is present or not."
+            ],
+            "color": "#432",
+            "bgcolor": "#000"
+        },
+        {
+            "id": 4,
+            "type": "LoadImage",
+            "pos": [
+                496.31859627609606,
+                -229.71277089860084
+            ],
+            "size": [
+                270,
+                314
+            ],
+            "flags": {},
+            "order": 1,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "image",
+                    "name": "image",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "image"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "choose file to upload",
+                    "name": "upload",
+                    "type": "IMAGEUPLOAD",
+                    "widget": {
+                        "name": "upload"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "IMAGE",
+                    "name": "IMAGE",
+                    "type": "IMAGE",
+                    "links": [
+                        25
+                    ]
+                },
+                {
+                    "localized_name": "MASK",
+                    "name": "MASK",
+                    "type": "MASK",
+                    "links": null
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "LoadImage"
+            },
+            "widgets_values": [
+                "cute-cat.jpg",
+                "image"
+            ]
+        },
+        {
+            "id": 8,
+            "type": "VLLMOmniDiffusionSampling",
+            "pos": [
+                465.5140218220927,
+                148.80072646828987
+            ],
+            "size": [
+                284.205078125,
+                226
+            ],
+            "flags": {},
+            "order": 2,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "n",
+                    "name": "n",
+                    "type": "INT",
+                    "widget": {
+                        "name": "n"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "num_inference_steps",
+                    "name": "num_inference_steps",
+                    "type": "INT",
+                    "widget": {
+                        "name": "num_inference_steps"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "guidance_scale",
+                    "name": "guidance_scale",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "guidance_scale"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "true_cfg_scale",
+                    "name": "true_cfg_scale",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "true_cfg_scale"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "vae_use_slicing",
+                    "name": "vae_use_slicing",
+                    "type": "BOOLEAN",
+                    "widget": {
+                        "name": "vae_use_slicing"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "vae_use_tiling",
+                    "name": "vae_use_tiling",
+                    "type": "BOOLEAN",
+                    "widget": {
+                        "name": "vae_use_tiling"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "seed",
+                    "name": "seed",
+                    "type": "INT",
+                    "widget": {
+                        "name": "seed"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "diffusion sampling params",
+                    "name": "diffusion sampling params",
+                    "type": "SAMPLING_PARAMS",
+                    "links": [
+                        27
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniDiffusionSampling"
+            },
+            "widgets_values": [
+                1,
+                50,
+                1,
+                1,
+                false,
+                false,
+                1992,
+                "randomize"
+            ]
+        },
+        {
+            "id": 14,
+            "type": "VLLMOmniWanParams",
+            "pos": [
+                518.6005521845747,
+                428.4455663719664
+            ],
+            "size": [
+                270,
+                106
+            ],
+            "flags": {},
+            "order": 3,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "guidance_scale_2",
+                    "name": "guidance_scale_2",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "guidance_scale_2"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "boundary_ratio",
+                    "name": "boundary_ratio",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "boundary_ratio"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "flow_shift",
+                    "name": "flow_shift",
+                    "type": "FLOAT",
+                    "widget": {
+                        "name": "flow_shift"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "Wan video params",
+                    "name": "Wan video params",
+                    "type": "VIDEO_PARAMS",
+                    "links": [
+                        28
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniWanParams"
+            },
+            "widgets_values": [
+                4,
+                0.875,
+                5
+            ]
+        },
+        {
+            "id": 12,
+            "type": "VLLMOmniGenerateVideo",
+            "pos": [
+                827.2566336087867,
+                -73.77449831827545
+            ],
+            "size": [
+                400,
+                346
+            ],
+            "flags": {},
+            "order": 4,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "image",
+                    "name": "image",
+                    "shape": 7,
+                    "type": "IMAGE",
+                    "link": 25
+                },
+                {
+                    "localized_name": "sampling_params",
+                    "name": "sampling_params",
+                    "shape": 7,
+                    "type": "SAMPLING_PARAMS",
+                    "link": 27
+                },
+                {
+                    "localized_name": "lora",
+                    "name": "lora",
+                    "shape": 7,
+                    "type": "REMOTE_LORA",
+                    "link": null
+                },
+                {
+                    "localized_name": "model_params",
+                    "name": "model_params",
+                    "shape": 7,
+                    "type": "VIDEO_PARAMS",
+                    "link": 28
+                },
+                {
+                    "localized_name": "url",
+                    "name": "url",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "url"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "model",
+                    "name": "model",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "model"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "prompt",
+                    "name": "prompt",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "prompt"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "negative_prompt",
+                    "name": "negative_prompt",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "negative_prompt"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "width",
+                    "name": "width",
+                    "type": "INT",
+                    "widget": {
+                        "name": "width"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "height",
+                    "name": "height",
+                    "type": "INT",
+                    "widget": {
+                        "name": "height"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "fps",
+                    "name": "fps",
+                    "type": "INT",
+                    "widget": {
+                        "name": "fps"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "num_frames",
+                    "name": "num_frames",
+                    "type": "INT",
+                    "widget": {
+                        "name": "num_frames"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [
+                {
+                    "localized_name": "video",
+                    "name": "video",
+                    "type": "VIDEO",
+                    "links": [
+                        26
+                    ]
+                }
+            ],
+            "properties": {
+                "Node name for S&R": "VLLMOmniGenerateVideo"
+            },
+            "widgets_values": [
+                "http://localhost:8000/v1",
+                "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+                "Make the cat in the attached video yawning. Eye closed and mouth wide open. Keep the original realistic style.",
+                "Cartoonish",
+                900,
+                1200,
+                12,
+                41
+            ]
+        },
+        {
+            "id": 13,
+            "type": "SaveVideo",
+            "pos": [
+                1257.5441369501657,
+                -44.12956394419863
+            ],
+            "size": [
+                374,
+                372.34896826553796
+            ],
+            "flags": {},
+            "order": 5,
+            "mode": 0,
+            "inputs": [
+                {
+                    "localized_name": "video",
+                    "name": "video",
+                    "type": "VIDEO",
+                    "link": 26
+                },
+                {
+                    "localized_name": "filename_prefix",
+                    "name": "filename_prefix",
+                    "type": "STRING",
+                    "widget": {
+                        "name": "filename_prefix"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "format",
+                    "name": "format",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "format"
+                    },
+                    "link": null
+                },
+                {
+                    "localized_name": "codec",
+                    "name": "codec",
+                    "type": "COMBO",
+                    "widget": {
+                        "name": "codec"
+                    },
+                    "link": null
+                }
+            ],
+            "outputs": [],
+            "properties": {},
+            "widgets_values": [
+                "video/ComfyUI",
+                "auto",
+                "auto"
+            ]
+        }
+    ],
+    "links": [
+        [
+            25,
+            4,
+            0,
+            12,
+            0,
+            "IMAGE"
+        ],
+        [
+            26,
+            12,
+            0,
+            13,
+            0,
+            "VIDEO"
+        ],
+        [
+            27,
+            8,
+            0,
+            12,
+            1,
+            "SAMPLING_PARAMS"
+        ],
+        [
+            28,
+            14,
+            0,
+            12,
+            3,
+            "VIDEO_PARAMS"
+        ]
+    ],
+    "groups": [
+        {
+            "id": 1,
+            "title": "Input",
+            "bounding": [
+                213.1706010087147,
+                -313.20095750667554,
+                560.8246471437027,
+                407.4246532472182
+            ],
+            "color": "#3f789e",
+            "font_size": 24,
+            "flags": {}
+        }
+    ],
+    "config": {},
+    "extra": {
+        "workflowRendererVersion": "LG",
+        "ds": {
+            "scale": 1.1469075819486894,
+            "offset": [
+                101.7632950847089,
+                492.29121889347806
+            ]
+        }
+    },
+    "version": 0.4
+}
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.25.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 # WORKDIR /vllm-workspace/vllm
@@ -26,10 +26,6 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation

-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.25.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 # WORKDIR /vllm-workspace/vllm
@@ -26,10 +26,6 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation

-RUN python3 -m pip install -v --no-deps \
-      "stepaudio2-minicpmo>=0.1.1" \
-      "step-audio2>=1.0.0"
-
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn

 ENTRYPOINT []
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.25.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,7 +35,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
      "psutil>=7.2.0" \
      "soundfile>=0.13.1" \
      "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
      "opencv-python>=4.12.0.88" \
      "imageio[ffmpeg]>=0.6.0" \
      "jiwer>=3.0.0" \
@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.25.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}

 ARG APP_DIR=/vllm-workspace/vllm-omni
@@ -35,7 +35,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
      "psutil>=7.2.0" \
      "soundfile>=0.13.1" \
      "pyttsx3>=2.99" \
-      "opencc==1.4.1" \
      "opencv-python>=4.12.0.88" \
      "imageio[ffmpeg]>=0.6.0" \
      "jiwer>=3.0.0" \
@@ -45,7 +44,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
      "hyperpyyaml>=1.2.0" \
      "onnx>=1.16.0" \
      "librosa>=0.10.0" \
-      "torchmetrics>=1.9.0" \
      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
      --default-timeout=60 && \
@@ -71,7 +71,8 @@ nav:
      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
      - LoRA: user_guide/diffusion/lora.md
      - Custom Pipeline: features/custom_pipeline.md
-      - Execution Modes: user_guide/diffusion/execution_modes.md
+      - Request-Level Batching: user_guide/diffusion/request_batching.md
+      - Step Execution: user_guide/diffusion/step_execution.md
    - Quantization:
      - Overview: user_guide/quantization/overview.md
      - Online Quantization: user_guide/quantization/online.md
@@ -101,49 +102,31 @@ nav:
  - CI: contributing/ci
  - Design Documents:
    - design/index.md
-    - Architecture Documents:
-      - Architecture Overview: design/architecture_overview.md
+    - design/architecture_overview.md
    - Feature Design:
-      - Runtime and Stage Execution:
-        - Disaggregated Inference: design/feature/disaggregated_inference.md
-        - Ray-based Execution: design/feature/ray_based_execution.md
-        - Async Chunk: design/feature/async_chunk.md
-        - Async Diffusion Output: design/feature/async_diffusion_output.md
-        - Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
-        - Automatic Prefix Caching: design/feature/prefix_caching.md
-      - Communication:
-        - OmniConnector Implementations:
-          - Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
-          - Mooncake Transfer Engine Connector: design/feature/omni_connectors/mooncake_transfer_engine_connector.md
-          - Mori Transfer Engine Connector: design/feature/omni_connectors/mori_transfer_engine_connector.md
-          - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
-          - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
-          - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
-      - Diffusion Acceleration:
-        - Parallelism:
-          - CFG-Parallel: design/feature/cfg_parallel.md
-          - Expert Parallel: design/feature/expert_parallel.md
-          - Hybrid Sharded Data Parallel (HSDP): design/feature/hsdp.md
-          - Pipeline Parallel: design/feature/pipeline_parallel.md
-          - Sequence Parallel: design/feature/sequence_parallel.md
-          - Tensor Parallel: design/feature/tensor_parallel.md
-          - VAE Patch Parallelism: design/feature/vae_parallel.md
-        - Attention Backends:
-          - Skip-Softmax: design/feature/skip_softmax.md
-        - Quantization: user_guide/quantization/overview.md
-        - Cache-DiT: design/feature/cache_dit.md
-        - TeaCache: design/feature/teacache.md
-        - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
-    - Infrastructure and Performance:
-      - Prometheus Metrics: design/metrics.md
-      - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
+      - design/feature/disaggregated_inference.md
+      - design/feature/ray_based_execution.md
+      - design/feature/omni_connectors/
+      - design/feature/prefix_caching.md
+      - design/feature/cfg_parallel.md
+      - design/feature/expert_parallel.md
+      - design/feature/sequence_parallel.md
+      - design/feature/tensor_parallel.md
+      - design/feature/vae_parallel.md
+      - design/feature/hsdp.md
+      - design/feature/cache_dit.md
+      - design/feature/teacache.md
+      - design/feature/skip_softmax.md
+      - design/feature/async_chunk.md
+      - design/feature/vae_parallel.md
+      - design/feature/diffusion_step_execution.md
+      - design/feature/diffusion_request_level_batching.md
+      - design/feature/diffusion_continuous_batching.md
    - Module Design:
-      - Runtime Modules:
-        - AR Module: design/module/ar_module.md
-        - DIT Module: design/module/dit_module.md
-      - Orchestration:
-        - Entrypoint Module: design/module/entrypoint_module.md
-        - AsyncOmni Architecture: design/module/async_omni_architecture.md
+      - design/module/ar_module.md
+      - design/module/dit_module.md
+      - design/module/entrypoint_module.md
+      - design/module/async_omni_architecture.md
  - Docs Guide: contributing/DOCS_GUIDE.md
 - API Reference:
  - api/README.md
@@ -52,13 +52,12 @@ vLLM-Omni is flexible and easy to use with:
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
- Full-duplex realtime serving with streaming audio input and output (experimental)

 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:

- Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
+- Omni-modality models (e.g. Qwen3-Omni, Cosmos3, HunyuanImage, BAGEL)
 - TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
+- Diffusion models — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
 - Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)

 For more information, checkout the following:
@@ -18,12 +18,18 @@ Specify the port:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```

-Load a custom deploy configuration with `--deploy-config`:
+For a migrated model, load a custom deploy configuration with `--deploy-config`:

 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --deploy-config /path/to/deploy_config.yaml
 ```

+The deprecated `--stage-configs-path` flag is retained for models that still use the legacy `stage_args` schema:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --stage-configs-path /path/to/legacy_stage_config.yaml
+```
+
 ## bench

 Run benchmark tests for online serving throughput.
@@ -8,6 +8,7 @@ The stage-based CLI is designed for deployments that require launching each pipe
 - For **migrated models** that utilize the bundled deployment YAML configurations located in
  `vllm_omni/deploy/`, the `--deploy-config` flag is only required to override the default configuration. By default, executing `vllm serve MODEL --omni ...`
  automatically loads the bundled deployment configuration.
+- For custom legacy `stage_args` YAMLs, pass the file with `--stage-configs-path`.

 Example: Initializing Stage 0 (Orchestrator and API Server):
 The commands below show a common device mapping where Stage 0 uses GPU 0 and
@@ -31,7 +32,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
    --omni-master-port 26000
 ```

-When utilizing a custom deployment YAML, append `--deploy-config /path/to/override.yaml` to each command execution.
+When utilizing a custom deployment YAML based on the new schema, append `--deploy-config /path/to/override.yaml` to each command execution. Conversely, for legacy models, substitute this parameter with `--stage-configs-path /path/to/stage_configs.yaml`.

 In the standard execution paradigm, the `--stage-overrides` argument is utilized to apply stage-specific configurations from a single CLI command.
 However, under the **stage-based CLI** paradigm, where each process strictly encapsulates a single stage, it is recommended to specify tuning parameters directly via discrete command-line flags for the respective stage, rather than constructing a composite `--stage-overrides` JSON string.
@@ -40,7 +40,6 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@Isotr0py](https://github.com/Isotr0py): Diffusion and Quantization
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
- [@NickCao](https://github.com/NickCao): CI, Tests, Entrypoint, and TTS Model Support
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
 - [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
 - [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
@@ -2,7 +2,7 @@

 This section lists the most common options for running vLLM-Omni.

-For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
+For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.16.0/configuration/index.html)

 Currently, the main options are maintained by stage configs for each model.

@@ -34,8 +34,12 @@ Two additional `StrategySpec` slots — `layer_hook_specs` and `kernel_specs`
 | Flag | Description |
 |------|-------------|
 | `--strategy-config PATH` | Path to a `strategy.yaml`. Only takes effect on registry-based deploy paths (`--deploy-config` or the bundled default deploy YAML). Its derived sizing is overlaid onto the merged stages *before* CLI overrides. |
+| `--stage-configs-path PATH` | **Legacy.** Loads the old `stage_args` YAML schema for un-migrated models. Passing `--strategy-config` together with this path is silently inapplicable — vLLM-Omni emits a warning and the strategy is ignored (see `vllm_omni/entrypoints/utils.py`, lines 372–380). |
 | `--omni-lb-policy POLICY` | Orchestrator-wide load-balancer policy. Strategy-derived from any `stage_replica` axis; an explicit CLI value wins and must match the derived one or `AsyncOmniEngine` raises `Conflicting load-balancer policy` at construction. |

+!!! warning
+    `--strategy-config` cannot be used with `--stage-configs-path` (the legacy deploy path). If your model has not been migrated to the registry-based pipeline yet, the strategy is silently dropped with a warning. Migrate to a registry-based model or use `--deploy-config` to apply a strategy.
+
 ## Sub-schemas

 ### `axis`
@@ -212,6 +216,17 @@ Strategy keys MUST be `model_stage` *names* (strings), the same labels stage con
 !!! warning "Mixing CLI overrides with a strategy on the same axis"
    A strategy is meant to be the single writer for the axes it declares. If a CLI override (`--tensor-parallel-size 2`, `--stage-overrides '{"0": {"tensor_parallel_size": 2}}'`) sets the same axis, the **CLI value wins** and a warning is emitted from `_reconcile_strategy_with_cli`. The device-layout guard then re-runs against the effective layout, so a CLI override that breaks `tp * dp * pp * num_replicas` will still fail fast at config time.

+!!! warning "Passing `--strategy-config` against the legacy `--stage-configs-path`"
+    Composable parallel only applies on the registry-based deploy path. If a model still resolves through the legacy `stage_args` YAML (loaded via `--stage-configs-path`), `--strategy-config` is silently inapplicable and emits:
+
+    ```text
+    --strategy-config (<path>) was provided but model '<model>' resolves via the
+    legacy stage_configs YAML path, which does not support composable-parallel
+    strategies; the strategy is ignored. Use a registry-based model to apply it.
+    ```
+
+    Migrate the model to the new deploy schema (see `configuration/stage_configs.md`) before applying a strategy.
+
 !!! warning "Conflicting `omni_lb_policy` across stages"
    `omni_lb_policy` is a single pipeline-wide knob. If two `stage_replica` axes in different stages derive different policies, `apply_strategy_specs` raises:

@@ -158,7 +158,7 @@ runtime:

 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-    --deploy-config /path/to/qwen3_omni_pd.yaml
+    --stage-configs-path /path/to/qwen3_omni_pd.yaml
 ```

 ## Operational Notes
@@ -1,9 +1,9 @@
-# Pipeline and deploy configurations
+# Deploy and legacy stage configurations

-In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run.
+In vLLM-Omni, a model's `PipelineConfig` defines its fixed stage topology, while a deploy configuration controls how those stages run. Models that have not migrated to this split still use legacy stage configurations with a `stage_args` schema.

 !!! note
-    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when `--deploy-config` is omitted. The model registry resolves the right pipeline and deploy YAML by `model_type`.
+    Default deploy config YAMLs (for example, `vllm_omni/deploy/qwen2_5_omni.yaml`, `vllm_omni/deploy/qwen3_omni_moe.yaml`, and `vllm_omni/deploy/qwen3_tts.yaml`) are bundled and loaded automatically when neither `--stage-configs-path` nor `--deploy-config` is provided — the model registry resolves the right pipeline + deploy YAML by `model_type`. Custom legacy `stage_args` YAMLs are still accepted via `--stage-configs-path`.

 ## Deploy configuration schema

@@ -86,9 +86,10 @@ stages:

 | Flag | Description |
 |------|-------------|
-| `--deploy-config PATH` | Load a deploy YAML. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
+| `--deploy-config PATH` | Load a new-schema deploy YAML. It is mutually exclusive with `--stage-configs-path`. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
 | `--stage-overrides JSON` | Per-stage JSON overrides, e.g. `'{"0":{"gpu_memory_utilization":0.5}}'`. Per-stage values always win over global flags. |
 | `--async-chunk` / `--no-async-chunk` | Flip the deploy YAML's `async_chunk:` bool. Unset (default) leaves the YAML value in force. |
+| `--stage-configs-path` | **Deprecated.** Load a legacy `stage_args` YAML. New-schema YAMLs passed to this flag are auto-detected for compatibility, but should use `--deploy-config`. It is mutually exclusive with `--deploy-config` and will be removed in a future release. |

 ### Stage-Based CLI Paradigm

@@ -118,7 +119,8 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
    --omni-master-port 26000
 ```

-When instantiating a custom deployment YAML, append the `--deploy-config /path/to/override.yaml` directive to all node invocations.
+When instantiating a custom deployment YAML conforming to the updated schema, append the `--deploy-config /path/to/override.yaml` directive
+to all node invocations. For legacy architectures (e.g., BAGEL) configured via deprecated `stage_args:` schemas, continue to specify the relevant configuration via `--stage-configs-path /path/to/config.yaml`.

 In the context of standard initialization architectures, utilizing the `--stage-overrides` parameter operates as the optimal methodology
 for delineating stage-specific tuning from the CLI interface:
@@ -218,7 +220,8 @@ Therefore, as a core part of vLLM-Omni, a model's pipeline and deployment config

 To override specific parameters, explicitly inject the customized configuration schema
 in both online and offline instantiation flows. Use the `--deploy-config` flag
-when loading a deploy configuration.
+when loading a deploy configuration, reserving the `--stage-configs-path` parameter
+exclusively to maintain compatibility with legacy `stage_args` YAML constructs.

 Examples:

@@ -233,6 +236,11 @@ For online serving:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
 ```

+Legacy online serving:
+
+```bash
+vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
 !!! important
    We are actively iterating on the definition of deployment configurations, and we welcome feedback from users and developers.

@@ -387,8 +395,8 @@ that support request-level batching with `step_execution` disabled.

 Use this together with `max_num_seqs > 1` for bursty serving traffic. `0`
 disables admission waiting and preserves the lowest first-request latency.
-For diffusion execution and batching tuning, see
-[Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
+For diffusion request-level batching tuning, see
+[Request-Level Batching](../user_guide/diffusion/request_batching.md).

 Default: `0.0`

@@ -44,11 +44,9 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
            "ignore_eos": true,
            "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
            "baseline": {
-                "H100": {
-                    "mean_ttft_ms": [500, 800],
-                    "mean_audio_ttfp_ms": [2000, 3500],
-                    "mean_audio_rtf": [0.25, 0.35]
-                }
+                "mean_ttft_ms": [500, 800],
+                "mean_audio_ttfp_ms": [2000, 3500],
+                "mean_audio_rtf": [0.25, 0.35]
            }
        }
    ]
@@ -144,6 +142,10 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
  --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
 pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
  --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+
+# Optional baseline assertion (default off)
+pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
+  --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 ```

 See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
@@ -200,4 +202,4 @@ You can add any benchmark running parameters you need here. For all optional par
 | num_prompts     | int / array | Yes      | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
 | request_rate    | float / array | No  | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
 | max_concurrency | int / array | No       | 1, [1, 2, 3]    | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode).                                                                                                                                                                                                             |
-| baseline        | object      | No       | see below       | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS.
+| baseline        | object      | No       | see above       | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).  
@@ -56,11 +56,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
      <td>/</td>
      <td><code>core_model and cpu</code></td>
      <td rowspan="2">&lt;15min</td>
-      <td>
-        <code>tests/{component}/…</code> mirroring <code>vllm_omni/{component}/</code><br>
-        (e.g. <code>tests/diffusion/</code>, <code>tests/engine/</code>, <code>tests/entrypoints/</code>)<br>
-        Do <strong>not</strong> add new top-level <code>tests/</code> dirs unrelated to a component.
-      </td>
+      <td>/tests/{component_name}/test_xxx</td>
      <td>
        <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
        Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
@@ -72,18 +68,16 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
      <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
      <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
      <td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
-      <td>High-priority features and component tests that require launching instances</td>
+      <td>High-priority features (using random lightweight models)</td>
      <td>High-priority interfaces (using random lightweight models)</td>
      <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
      <td>
-        <strong>Model E2E:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}.py</code><br>
-        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
-        <strong>Feature integration:</strong><br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
-        (e.g. <code>fullduplex/</code>, <code>custom_pipeline/</code>, <code>rlhf_test/</code>)<br>
-        <strong>Component / interface Test (GPU):</strong><br>
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
      </td>
      <td>
        <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
@@ -96,19 +90,18 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
      <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
      <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
      <td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
-      <td>Medium-priority features and component tests that require launching instances</td>
+      <td>Medium-priority features (using random lightweight models)</td>
      <td>Medium-priority interfaces (using random lightweight models)</td>
      <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
      <td>&lt;30min</td>
      <td>
-        <strong>Model E2E:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}.py</code><br>
-        <code>tests/e2e/offline_inference/test_{model}.py</code><br>
-        <code>tests/e2e/accuracy/</code><br>
-        <strong>Feature integration:</strong><br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
-        <strong>Component / interface Test:</strong><br>
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
+        /tests/e2e/offline_inference/test_{model_name}.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
      </td>
      <td>
        <a href="test_writing_guide.md#l3-level-testing-core-integration-performance-and-accuracy-verification">L3</a><br>
@@ -121,23 +114,26 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
      <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
      <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
      <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
-      <td>Low-priority features and component tests that require launching instances</td>
+      <td>Low-priority features (using real weights)</td>
      <td>Low-priority interfaces (using real weights)</td>
      <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
      <td>&lt;3 hour</td>
      <td>
-        <strong>Model E2E:</strong><br>
-        <code>tests/e2e/online_serving/test_{model}_expansion.py</code><br>
-        <code>tests/e2e/offline_inference/test_{model}_expansion.py</code><br>
-        <code>tests/e2e/accuracy/test_{model}.py</code><br>
-        <strong>Feature integration:</strong><br>
-        <code>tests/e2e/features/&lt;feature&gt;/</code><br>
-        <strong>Component / interface Test:</strong><br>
-        <code>tests/{component}/…</code>, <code>tests/entrypoints/…</code><br>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx<br>
        <strong>Performance:</strong><br>
-        <code>tests/dfx/perf/tests/</code><br>
-        <strong>Doc examples:</strong><br>
-        <code>tests/examples/online_serving/</code>, <code>tests/examples/offline_inference/</code>
+        /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),<br>
+        test_voxcpm2.json, test_higgs_audio_v3.json, and<br>
+        /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
+        <strong>Doc Test:</strong><br>
+        tests/examples/online_serving/test_{model_name}.py<br>
+        tests/examples/offline_inference/test_{model_name}.py<br>
+        <strong>Accuracy Test:</strong><br>
+        /tests/e2e/accuracy/test_{model_name}.py
      </td>
      <td>
        <a href="test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing">L4</a><br>
@@ -156,9 +152,11 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
      <td> Depends on reality </td>
      <td>
        <strong>Stability:</strong><br>
-        <code>tests/dfx/stability/tests/</code><br>
+        /tests/dfx/stability/tests/test_qwen3_omni.json<br>
+        /tests/dfx/stability/tests/test_wan22.json<br>
        <strong>Reliability:</strong><br>
-        <code>tests/dfx/reliability/test_reliability_{model}.py</code>
+        tests/dfx/reliability/test_reliability_{model_key}.py<br>
+        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
      </td>
      <td>
        <a href="test_writing_guide.md#l5-level-testing-stability-and-reliability-testing">L5</a><br>
@@ -170,9 +168,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
  </tbody>
 </table>

-**Test Dir placement (summary):** component / unit under `tests/{component}/` mirroring `vllm_omni/`; model E2E under `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, and `tests/e2e/accuracy/`; feature integration under `tests/e2e/features/<feature>/`; doc example tests under `tests/examples/online_serving/` and `tests/examples/offline_inference/`; performance under `tests/dfx/perf/`; stability under `tests/dfx/stability/`; reliability under `tests/dfx/reliability/`. Do **not** add new top-level directories under `tests/` that are unrelated to a `vllm_omni` component (or to the established `e2e` / `dfx` / `helpers` / `examples` / `buildkite` layout).
-
-For per-level test authoring (markers, examples), see [Test Writing Guide](./test_writing_guide.md).
+For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md).

 ## Common Specifications

@@ -4,20 +4,137 @@ This document describes how to author tests for each CI level (L1–L5). For the

 ## Test Directory Structure

-Canonical placement by CI level is in the **Test Dir** column of the level matrix in [Test System Overview](./test_system_overview.md).
+Legend: `✅` = test exists, `⬜` = suggested to add.
+```
+vllm_omni/                                    tests/
+├── config/                             →     ├── config/
+│   ├── model.py                              │   └── test_model.py                    ⬜
+│   └── lora.py                               │   └── test_lora.py                      ⬜
+│
+├── core/                               →     ├── core/
+│   └── sched/                                 │   └── sched/
+│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
+│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
+│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
+│
+├── diffusion/                          →     ├── diffusion/
+│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
+│   ├── attention/                             │   ├── attention/
+│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
+│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
+│   ├── distributed/                           │   ├── distributed/
+│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
+│   │                                            │   │   ├── test_cfg_parallel.py        ✅
+│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
+│   ├── lora/                                   │   ├── lora/
+│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
+│   │                                            │   │   └── test_lora_manager.py        ✅
+│   ├── models/                                 │   ├── models/
+│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
+│   │   ├── z_image/                            │   │   └── z_image/
+│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
+│   └── worker/                                 │   └── worker/
+│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
+│       └── diffusion_model_runner.py            │
+│
+├── distributed/                         →     ├── distributed/
+│   └── omni_connectors/                         │   └── omni_connectors/
+│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
+│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
+│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
+│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
+│
+├── engine/                             →     ├── engine/
+│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
+│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
+│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
+│
+├── entrypoints/                        →     ├── entrypoints/
+│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
+│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
+│   │   └── ...                                  │   │   └── test_*.py                  ⬜
+│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
+│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
+│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
+│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
+│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
+│
+├── inputs/                             →     ├── inputs/
+│   ├── data.py                                 │   ├── test_data.py                   ⬜
+│   ├── parse.py                                │   ├── test_parse.py                 ⬜
+│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
+│
+├── model_executor/                     →     ├── model_executor/
+│   ├── layers/                                  │   ├── layers/
+│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
+│   ├── model_loader/                            │   ├── model_loader/
+│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
+│   ├── models/                                  │   ├── models/
+│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
+│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
+│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
+│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
+│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
+│   │       └── ...                               │   │   └── qwen3_omni/
+│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
+│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
+│   └── stage_input_processors/                  │   └── stage_input_processors/
+│       └── ...                                  │       └── test_*.py                 ⬜
+│
+├── sample/                             →     ├── sample/
+│   └── __init__.py                             │   └── test_*.py                      ⬜
+│
+├── utils/                              →     ├── utils/
+│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
+│
+├── worker/                             →     ├── worker/
+│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
+│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
+│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
+│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
+│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
+│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
+│
+├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
+│   ├── cuda/
+│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
+│   ├── rocm/
+│   └── xpu/worker/
+│
+├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
+├── (logger, patch, request, version)    →     (no corresponding unit test)
+│
+└── e2e (tests side only)               →     ├── e2e/
+                                               ├── online_serving/                     ✅ non-empty
+                                               │   ├── test_qwen2_5_omni.py
+                                               │   ├── test_async_omni.py
+                                               │   ├── test_qwen3_omni.py
+                                               │   ├── test_qwen3_omni_expansion.py
+                                               │   ├── test_mimo_audio.py
+                                               │   └── test_images_generations_lora.py
+                                               └── offline_inference/                  ✅
+                                                   ├── test_qwen2_5_omni.py
+                                                   ├── test_qwen3_omni.py
+                                                   ├── test_bagel_text2img.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
+                                                   ├── test_zimage_tensor_parallel.py
+                                                   ├── test_cache_dit.py
+                                                   ├── test_teacache.py
+                                                   ├── test_stable_audio_expansion.py
+                                                   ├── test_diffusion_cpu_offload.py
+                                                   ├── test_diffusion_layerwise_offload.py
+                                                   ├── test_diffusion_lora.py
+                                                   ├── test_sequence_parallel.py
+                                                   └── stage_configs/                  (legacy schema, still
+                                                       ├── bagel_*.yaml                 present for unmigrated
+                                                       └── npu/, rocm/, etc.            models)

-Summary:
-
- **Component / unit:** `tests/{component}/…` mirroring `vllm_omni/{component}/`
- **Model E2E:** `tests/e2e/online_serving/`, `tests/e2e/offline_inference/`, `tests/e2e/accuracy/`
- **Feature integration:** `tests/e2e/features/<feature>/`
- **Doc example tests:** `tests/examples/online_serving/`, `tests/examples/offline_inference/`
- **Performance:** `tests/dfx/perf/tests/` (JSON configs; runners under `tests/dfx/perf/scripts/`)
- **Stability:** `tests/dfx/stability/tests/`
- **Reliability:** `tests/dfx/reliability/`
- Do **not** add new top-level `tests/` directories unrelated to a component (or to `e2e` / `dfx` / `helpers` / `examples` / `buildkite`)
-
-Deploy YAMLs: `vllm_omni/deploy/` via `tests.helpers.stage_config.get_deploy_config_path` (see [stage configs](../../configuration/stage_configs.md)).
+# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under
+# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md.
+```

 ## Markers for Tests

@@ -256,13 +373,13 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that

 #### 1.3 Test Directory and Execution Files

-A clear directory structure is key to managing test cases efficiently. See the **Test Dir** column in [Test System Overview](./test_system_overview.md) for the canonical placement rules.
+A clear directory structure is key to managing test cases efficiently.

-   ***L1 (component / unit)***: `tests/{component_name}/…` mirroring `vllm_omni/{component_name}/` (e.g. `tests/diffusion/`, `tests/entrypoints/`). Do **not** invent new top-level `tests/` folders unrelated to a component.
-   ***L2 (basic model E2E)***:
-    -   Online serving: `tests/e2e/online_serving/test_{model_name}.py`
-    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}.py`
-   ***Feature-level integration*** (cross-cutting, not a single-model smoke): `tests/e2e/features/<feature>/` (e.g. `fullduplex/`, `custom_pipeline/`, `rlhf_test/`).
+-   ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py`
+-   -   Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file.
+-   ***L2 Test Directory***:
+-   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
+    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`

 #### 1.4 Execution Method and Example

@@ -368,12 +485,10 @@ L3 level testing executes after code is merged into the main branch. Its core pu

 #### 2.3 Test Directory and Execution Files

-   ***Functional / model E2E***:
-    -   Online serving: `tests/e2e/online_serving/test_{model_name}_expansion.py`
-    -   Offline inference: `tests/e2e/offline_inference/test_{model_name}_expansion.py`
-    -   Accuracy / similarity: `tests/e2e/accuracy/`
-    -   (`_expansion.py` holds broader scenario coverage than the L2 smoke file.)
-   ***Feature-level integration***: `tests/e2e/features/<feature>/` (not under a new top-level `tests/` directory).
+-   ***Functional Testing***:
+-   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
+    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
+    -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).

 #### 2.4 Execution Method and Example

@@ -653,7 +653,7 @@ python end2end.py --model your-org/your-model-name --modality text2img --prompts
 Mirror BAGEL’s online serving setup:

 - Server launcher: `examples/online_serving/your_model_name/run_server.sh`
-  - Wrap `vllm serve ... --omni --port ...` (and `--deploy-config ...` if needed)
+  - Wrap `vllm serve ... --omni --port ...` (and `--stage-configs-path ...` if needed)
 - Client: `examples/online_serving/your_model_name/openai_chat_client.py`
  - Send requests to `POST /v1/chat/completions`
  - Support multimodal inputs (e.g., base64 image) if your model needs it
@@ -755,8 +755,7 @@ omni = Omni(model="your-model", ulysses_degree=2, ring_degree=2)

 ### Step Execution

-See the detailed
-[Diffusion Continuous Batching design guide](../../design/feature/diffusion_continuous_batching.md).
+See detailed design guide: [How to add step execution support](../../design/feature/diffusion_step_execution.md)

 Use this only when your pipeline can be split into stable request-scoped and
 step-scoped phases. The reference implementation is
@@ -770,8 +769,9 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.

-The same design guide covers the experimental batched step-wise path used when
-`max_num_seqs > 1`.
+If you want the pipeline to work with the experimental batched step-wise path
+(`max_num_seqs > 1`), also see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

 If you expose this in example scripts or recipes, keep it opt-in. Surface
 runtime features like `step_execution` as optional flags instead of silently
@@ -48,8 +48,8 @@ vllm_omni/deploy/
 ```

 New in-tree models should register their `PipelineConfig` in
-`vllm_omni/config/pipeline_registry.py`; use `--deploy-config` for custom
-deployment YAMLs.
+`vllm_omni/config/pipeline_registry.py`; use `--stage-configs-path` only for
+custom legacy `stage_args` YAMLs.

 ## Step-by-Step Implementation

@@ -449,8 +449,9 @@ Key configuration fields:
 !!! note
    New in-tree models should define frozen topology in `models/<model>/pipeline.py`,
    register it in `vllm_omni/config/pipeline_registry.py`, and put deployment
-    knobs in `vllm_omni/deploy/<model>.yaml`. Use `--deploy-config` to load a
-    custom deployment YAML.
+    knobs in `vllm_omni/deploy/<model>.yaml`. Legacy `stage_args` YAMLs are still
+    accepted for custom configs via `--stage-configs-path`, but should not be used
+    for new bundled defaults.

 ### Batch mode

@@ -1,174 +0,0 @@
-# Async Diffusion Output
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Performance](#performance)
-3. [Architecture](#architecture)
-4. [Model Coverage](#model-coverage)
-5. [Configuration](#configuration)
-6. [Related Files](#related-files)
-
-## Overview
-
-The async diffusion output feature moves the D2H (Device→Host) copy and SHM packing from the Worker main thread to a background daemon thread, so the GPU default stream can immediately start the next request's forward pass. This eliminates the GPU bubble caused by synchronous D2H/packing.
-
-**Core benefit**: the previous request's output D2H/packing overlaps with the next request's forward pass.
-
-**Automatically enabled** when `step_execution=False` (default, request-mode). No extra configuration needed. When `step_execution=True` (step-mode), the original synchronous path is used — pump thread and Worker background thread are not started.
-
-## Performance
-
-### HunyuanImage-3.0 (TP4)
-
-| Resolution | Async Off (QPS) | Async On (QPS) | Change |
-|---|---|---|---|
-| 1024×1024 | 0.4773 | 0.4802 | +0.60% |
-| 768×768 | 0.8370 | 0.8533 | +1.95% |
-
-**Analysis**: smaller resolution → larger QPS gain. D2H/packing is relatively fixed overhead; forward is variable (scales with resolution). Smaller images → shorter forward → D2H/packing occupies a larger fraction of total time → more benefit from overlapping it.
-
-## Architecture
-
-### Execution Timeline
-
-```
-Before (synchronous D2H):
-[forward req1] [D2H+SHM pack req1] [forward req2] [D2H+SHM pack req2]
-                                 ^^^^^^^^^^^^^^^^ GPU bubble
-
-After (async D2H):
-[forward req1] [forward req2]
-[D2H+SHM pack req1 (side stream)] [D2H+SHM pack req2 (side stream)]
-                                 ^^^^^^^^^^^^^^^^ bubble eliminated
-```
-
-### Data Flow
-
-```
-                    ┌─────────────────────────────────────────────────────┐
-                    │                 Worker Process                      │
-                    │                                                    │
-  execute_model ──> │ WorkerBusyLoop ──> forward() ──> DiffusionOutput   │
-                    │       │                              │ (GPU tensor)│
-                    │       │                    ┌─────────┴──────────┐  │
-                    │       │                    │  async output path │  │
-                    │       │                    │                    │  │
-                    │       │              compute_done ──> result_mq │  │
-                    │       │                    │                    │  │
-                    │       │                    ▼                    │  │
-                    │       │            AsyncOutputThread            │  │
-                    │       │         (side CUDA stream)             │  │
-                    │       │         wait_event(gpu_event)          │  │
-                    │       │         pack_diffusion_output_shm()    │  │
-                    │       │         D2H + SHM write                │  │
-                    │       │                    │                    │  │
-                    │       │              output_ready ──> result_mq │  │
-                    │       │                    │                    │  │
-                    │       ▼                    ▼                    │  │
-                    │  (dequeue next request)                        │  │
-                    └─────────────────────────────────────────────────────┘
-                                         │
-                                         ▼
-                    ┌─────────────────────────────────────────────────────┐
-                    │              Executor (main process)               │
-                    │                                                    │
-                    │  ResultPumpThread (sole reader of result_mq)       │
-                    │       │                                            │
-                    │       ├── AsyncDiffusionOutput(COMPUTE_DONE)       │
-                    │       │   → resolve _rpc_futures[rpc_id]           │
-                    │       │   → RunnerOutput(async_output_id=...)      │
-                    │       │                                            │
-                    │       ├── AsyncDiffusionOutput(OUTPUT_READY)       │
-                    │       │   → resolve _output_futures[async_output_id]│
-                    │       │   → or batch split via _batch_split_map   │
-                    │       │                                            │
-                    │       └── Non-async message                        │
-                    │           → _sync_result_buffer (for other RPCs)   │
-                    │                                                    │
-                    │  wait_output_ready(async_output_id) → Future       │
-                    └─────────────────────────────────────────────────────┘
-```
-
-### Key Components
-
-1. **AsyncDiffusionOutput** (`data.py`): Protocol envelope for `result_mq`. `kind` field routes messages:
-   - `COMPUTE_DONE` — forward finished, GPU can start next request
-   - `OUTPUT_READY` — D2H/SHM packing finished, final output available
-   - `RPC_RESULT` — ordinary RPC return (including error propagation)
-
-2. **Worker AsyncOutputThread** (`diffusion_worker.py`): Background daemon thread that:
-   - Waits for `gpu_event` (cross-stream sync ensuring forward wrote the tensor)
-   - Runs `pack_diffusion_output_shm()` on a side CUDA stream
-   - Enqueues `OUTPUT_READY` to `result_mq` when packing completes
-
-3. **ResultPumpThread** (`multiproc_executor.py`): Sole reader of `result_mq` in request-mode. Dispatches messages:
-   - `COMPUTE_DONE` / `RPC_RESULT` → resolves `_rpc_futures[rpc_id]`
-   - `OUTPUT_READY` → resolves `_output_futures[async_output_id]` (with batch split via `_batch_split_map`)
-   - Non-async messages → `_sync_result_buffer` for other RPCs
-
-4. **collective_rpc() two-path dispatch** (`multiproc_executor.py`):
-   - **Path 1**: `execute_model` / `execute_model_batch` → generates `rpc_id`, registers Future, waits for pump to deliver `compute_done`
-   - **Path 2**: All other RPCs → `_sync_result_buffer` (pump-fed) or `_result_mq` (step-mode, no pump)
-
-5. **record_device_event** (`platforms/`): Cross-platform GPU event recording for side-stream synchronization. Implemented for CUDA, NPU, ROCm, XPU, MUSA; base class returns `None` (safe no-op).
-
-6. **IPC side-stream D2H** (`ipc.py`): `pack_diffusion_output_shm()` and helpers accept `d2h_stream` parameter; use `pin_memory` + `copy_(non_blocking=True)` on the side stream instead of synchronous `.cpu()`.
-
-### Batch Split
-
-When `execute_model_batch` returns a `COMPUTE_DONE` with a single `async_output_id` for the entire batch, the executor splits it into per-request `async_output_id`s (format `{batch_id}/{request_id}`) via `_batch_split_map`. When `OUTPUT_READY` arrives, the pump extracts per-request results from the batch output and resolves each request's output future independently.
-
-## Model Coverage
-
-### Currently Supported (request-mode, `step_execution=False`)
-
-All models running in request-mode (`step_execution=False`, the default) automatically use async output.
-
-**Verified models**:
-
-| Model | Type | `supports_request_batch` |
-|---|---|---|
-| **HunyuanImage-3.0** | Image | `False` |
-| **Qwen-Image** | Image | `True` |
-
-Other models with `step_execution=False` are also supported but not yet verified.
-
-### Not Yet Supported (step-mode, `step_execution=True`)
-
-When `step_execution=True` (or `streaming_output=True`, which auto-enables step-mode), models use `execute_stepwise` instead of `execute_model`, which is not in the async Path 1 whitelist. Async output is not applicable.
-
-**Verified**: Helios (`step_execution=True`, does not benefit from this feature).
-
-Other models with `step_execution=True` are also not applicable.
-
-Adapting step-mode requires additional design because:
-
- Each step's `RunnerOutput` must be available synchronously for the scheduler to advance
- Intermediate latent tensors may not need D2H (only the final step needs D2H + postprocess)
- Steps are serial (step N+1 depends on step N output), so the overlap benefit is limited to "final step's D2H overlaps with the next request's forward"
-
-**Planned approach**: first cover only the final step's async D2H, keep intermediate steps synchronous. This matches the request-mode benefit pattern.
-
-## Configuration
-
-No configuration needed. Async output is automatically enabled when `step_execution=False` (default).
-
-| `step_execution` | Mode | Async Output | Pump Thread | Worker BG Thread |
-|---|---|---|---|---|
-| `False` (default) | request-mode | ✅ enabled | ✅ started | ✅ started |
-| `True` | step-mode | ❌ disabled | ❌ not started | ❌ not started |
-
-## Related Files
-
- `vllm_omni/diffusion/data.py`: `AsyncDiffusionOutput`, `AsyncOutputKind`, `DiffusionOutput.async_output_id`
- `vllm_omni/diffusion/worker/diffusion_worker.py`: `WorkerProc._return_result()`, `_async_output_loop()`, `_generate_async_output_id()`
- `vllm_omni/diffusion/executor/multiproc_executor.py`: `ResultPumpThread`, `collective_rpc()` two-path dispatch, `wait_output_ready()`, `_batch_split_map`
- `vllm_omni/diffusion/diffusion_engine.py`: `step()` / `step_streaming()` / `add_req_and_wait_for_response()` async output waiting
- `vllm_omni/diffusion/sched/request_scheduler.py`: `update_from_output()` — `async_output_id` → `FINISHED_COMPLETED`
- `vllm_omni/diffusion/ipc.py`: `pack_diffusion_output_shm()` side-stream D2H path
- `vllm_omni/platforms/interface.py`: `OmniPlatform.record_device_event()` base class
- `vllm_omni/platforms/cuda/platform.py`: `CudaOmniPlatform.record_device_event()`
- `vllm_omni/platforms/npu/platform.py`: `NPUOmniPlatform.record_device_event()`
- `vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py`: `get_hunyuan_image3_post_process_func()`, postprocess removal
- `vllm_omni/diffusion/registry.py`: `_DIFFUSION_POST_PROCESS_FUNCS` postprocess registry
@@ -1,232 +1,165 @@
-# Diffusion Continuous Batching
+# Continuous Batching for Step-Wise Diffusion

-This document describes the unified diffusion execution architecture, including
-request execution, request-level batching, step execution, continuous batching,
-and output delivery. For user-facing configuration and CLI examples, see
-[Diffusion Execution Modes](../../user_guide/diffusion/execution_modes.md).
+!!! warning "Experimental Feature"
+    This feature is experimental. It currently applies only to native
+    diffusion pipelines running with `step_execution=True`.
+
+This document describes the batching extension built on top of
+[Diffusion Step Execution](diffusion_step_execution.md). The base
+step-execution contract is unchanged. The batching work is mainly in the
+scheduler and runner layers.
+
+## Why It Helps
+
+Step-wise execution breaks a long denoise loop into scheduler-visible units.
+That gives the runtime a place to admit other compatible requests between
+steps instead of waiting for an entire request to finish.
+
+This matters most in low-MFU or bursty serving scenarios:
+
+- one request's denoise step may not fully saturate the GPU
+- several compatible requests can share the same denoise forward pass
+- throughput and device utilization can improve without changing request-local
+  scheduler state
+
+This is **not** a guaranteed single-request latency win. The main benefit is
+usually higher utilization and better throughput when the workload contains
+multiple in-flight compatible requests.

 ## Overview

-`DiffusionEngine` selects an execution mode from `step_execution` and uses
-`max_num_seqs` as that mode's scheduler capacity:
+With continuous batching enabled:

-| Configuration | Engine mode | Scheduler | Execution |
-|---|---|---|---|
-| `step_execution=False`, `max_num_seqs=1` | `REQUEST_BATCH` | `RequestScheduler` | One complete request-level `forward()` |
-| `step_execution=False`, `max_num_seqs>1` | `REQUEST_BATCH` | `RequestScheduler` | One fused `forward()` over compatible requests |
-| `step_execution=True`, `max_num_seqs=1` | `STEP_BATCH` | `StepScheduler` | One request advanced one denoise step per scheduler tick |
-| `step_execution=True`, `max_num_seqs>1` | `STEP_BATCH` | `StepScheduler` | Compatible requests advanced together in step waves |
+- the scheduler may keep multiple compatible requests active at the same time
+- the runner packs request-local step state into one `InputBatch`
+- `denoise_step()` runs on that batch
+- `step_scheduler()` and `post_decode()` still run per request

-Serial request execution is handled inside `REQUEST_BATCH`; it is not a
-separate engine mode. Similarly, continuous batching is the multi-request
-configuration of `STEP_BATCH`, not an independent execution mode.
+The current implementation is conservative:

-The engine performs the following initialization:
+- only compatible requests are batched together
+- per-request progress and completion remain independent

-1. Resolve model-specific pre- and post-processing hooks.
-2. Select `REQUEST_BATCH` or `STEP_BATCH`.
-3. Construct the configured `DiffusionExecutor`.
-4. Construct `RequestScheduler` or `StepScheduler`, unless a scheduler was
-   explicitly injected.
-5. Bind `execute_batch` or `execute_step`.
+Here, "continuous batching" means the step-wise path enabled by
+`step_execution=True`. Request-mode `DiffusionRequestBatch` is static
+request-level batching for one full pipeline `forward()` call; it does not
+admit or remove requests between denoise steps.

-`DiffusionEngine.make_engine()` owns startup warmup after engine construction.
-This keeps construction, backend selection, and warmup failure cleanup in one
-lifecycle.
+## Enablement

-## Unified Output Stream
+Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
+above `1` if you want batching:

-All asynchronous diffusion requests use the same per-request output stream,
-regardless of whether the caller exposes streaming results to the user.
-
-```python
-async for output in engine.step_streaming(request):
-    ...
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
 ```

-When a request is admitted, the engine creates an
-`asyncio.Queue[DiffusionOutput]` keyed by its request ID. The busy loop sends
-both intermediate chunks and final results through that queue:
+`--max-num-seqs 1` keeps the step-wise path without enabling batching.

-```text
-request
-  -> scheduler.add_request()
-  -> scheduler.schedule()
-  -> execute_batch() or execute_step()
-  -> scheduler.update_from_output()
-  -> per-request output queue
-  -> step_streaming()
-```
+For a reproducible replay flow using the bundled serving benchmark, see the
+Qwen-Image replay commands in
+[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
+and
+[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).

-Streaming callers forward every yielded output. Non-streaming callers consume
-the same generator to completion and return only its final output. This avoids
-maintaining separate future-based and queue-based result paths.
+## Scheduler

-`DiffusionEngine.step()` and
-`async_add_req_and_wait_for_response()` remain deprecated compatibility
-wrappers. New integrations should consume `step_streaming()` or
-`async_add_req_and_stream_response()`.
+The scheduler derives its batch capacity from `max_num_seqs` through
+`max_num_running_reqs`.

-### Completion and Cancellation
+Batch admission is gated by
+[`StepBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
+which is built from shape-sensitive and CFG-sensitive sampling fields. This is
+the core correctness rule for batching: requests are only co-batched when they
+share the same denoise tensor contract.

-Output delivery and scheduler lifecycle are related but separate:
+There are three important details:

- A terminal scheduler result always finalizes request state.
- A disconnected or cancelled consumer removes only its delivery queue.
- The scheduler may still finish and clean up the request after its queue has
-  been removed.
- Engine shutdown sends an error output to all remaining streams.
+- `num_inference_steps` is not part of the key, so requests with different
+  total step counts can still share a batch
+- requests also do not need to be at the same current denoise progress; active
+  requests can continue batching even when their current step indices diverge
+- admission is still FIFO, so an incompatible request at the head of the
+  waiting queue blocks later compatible requests

-This separation prevents cancelled consumers from leaving active request IDs in
-the scheduler.
+Today that compatibility rule is still shape-sensitive. `height`, `width`,
+`num_frames`, and CFG-related fields remain part of the key, so different
+resolutions or incompatible guidance settings do **not** co-batch yet. The
+key also covers LoRA identity (`lora_int_id`, `lora_scale`), so requests
+targeting different adapters or scales run in separate batches and the
+worker can activate exactly one adapter per step.

-## Request-Batch Execution
+The scheduler batching unit is one logical `OmniDiffusionRequest`. In the
+step-wise path, runtime tensor batching is represented as `StepInputBatch`. For
+request-mode prompt semantics, see
+[Request-Level Batching](../../user_guide/diffusion/request_batching.md).

-Request-batch execution runs the complete diffusion pipeline for each scheduler
-wave. Each `OmniDiffusionRequest` remains an independent logical request with
-its own prompt, sampling parameters, seed, request ID, output, error, and abort
-state.
+## Runner

-With `max_num_seqs=1`, `execute_batch()` handles the serial request path. With a
-larger value, a pipeline can opt into fused execution by declaring:
+The runner keeps persistent per-request execution state in
+[`StepRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
+while the scheduler owns a separate lightweight
+[`SchedulerRequestState`](gh-file:vllm_omni/diffusion/sched/interface.py)
+for queueing and lifecycle tracking.

-```python
-supports_request_batch = True
-```
+For each step, the runner builds an
+[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
+active request states:

-Its `forward()` method must accept `DiffusionRequestBatch` and return one
-`DiffusionOutput` per request. The runner validates the result count and maps
-outputs back to their original request IDs with `BatchRunnerOutput`.
+- prompt embeddings and masks are normalized and padded
+- dynamic tensors such as `latents` and `timesteps` are gathered each step
+- buffers are reused when batch composition stays the same

-Current in-tree request-batch implementations include the Flux, LTX-2, SD3, and
-Qwen-Image pipelines. Treat the pipeline capability flag in the source as
-authoritative because support continues to expand.
+The step-wise batched path is:

-### Request-Batch Data Flow
-
- `DiffusionSchedulerOutput` carries scheduled request IDs and payloads.
- `DiffusionRequestBatch` provides the pipeline-facing static batch.
- `BatchRunnerOutput` carries per-request results back to the engine.
-
-Request-local setup, such as seed handling and output/error mapping, remains
-per request. Cache refresh, homogeneous LoRA activation, and
-`pipeline.forward(req_batch)` can be shared by the batch.
-
-### Compatibility and Admission
-
-`RequestBatchSamplingParamsKey` contains fields that affect the tensor contract,
-including shape, guidance, output count, and LoRA identity. Only compatible
-requests can share a fused forward call.
-
-Admission is FIFO and conservative. An incompatible request at the front of the
-waiting queue can prevent later compatible requests from joining the current
-wave.
-
-`request_batch_max_wait_ms` optionally delays the first schedule operation of a
-new wave so bursty arrivals can accumulate. It applies only when:
-
- the pipeline supports request batching;
- step execution is disabled;
- no requests are currently running; and
- the configured wait is greater than zero.
-
-The wait ends when the queue reaches capacity, stabilizes briefly, reaches its
-deadline, or the engine stops.
-
-## Step-Batch Execution
-
-Step execution exposes denoising progress to the scheduler. A supporting
-pipeline implements four stateful operations:
-
-| Operation | Responsibility |
-|---|---|
-| `prepare_encode(state)` | Validate input, encode prompts, initialize latents and timesteps, and create request-local scheduler state |
-| `denoise_step(input_batch, *, states=...)` | Run one denoise forward for the scheduler-provided request states |
-| `step_scheduler(state, noise_pred)` | Update latents and advance request progress |
-| `post_decode(state)` | Decode and postprocess a completed request or output boundary |
-
-Persistent request state lives in `StepRequestState`. Pipeline-specific fields
-that do not belong in the shared contract should be stored in `state.extra`.
-Queueing and lifecycle metadata remain in the scheduler's request state.
-
-Current native pipelines that explicitly enable step execution include
-Qwen-Image, HunyuanImage3, and Helios. Step execution alone does not imply
-continuous-batching support: Qwen-Image accepts batched step states.
-HunyuanImage3 accepts batched step states only when its resolved self-attention
-backend is `TORCH_SDPA`; otherwise it rejects groups larger than one request.
-Configure `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or
-`diffusion_attention_config.default.backend=TORCH_SDPA` when
-`max_num_seqs>1`. See the
-[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
-for its validated configuration. Helios supports only a single active step
-request and must use `max_num_seqs=1`.
-
-### Continuous Batching
-
-When `max_num_seqs>1`, `StepScheduler` can keep multiple compatible requests
-active. The runner gathers their state into `InputBatch`, performs one batched
-denoise forward, then applies scheduler updates per request:
-
-1. Receive transferred KV payloads, then run `prepare_encode()` for newly
-   admitted requests.
+1. Run `prepare_encode()` for newly admitted requests.
 2. Build or refresh `InputBatch`.
-3. Run one batched `denoise_step(input_batch, states=states)`.
-4. Slice noise predictions back to each request.
-5. Run each request's `step_scheduler()`.
-6. Run `post_decode()` at a chunk boundary or request completion.
-7. Scatter updated latents back into persistent request state.
+3. Run one batched `denoise_step(input_batch)`.
+4. Slice the batched `noise_pred` back per request.
+5. Run per-request `step_scheduler()`.
+6. Run `post_decode()` only for requests that finished denoising.
+7. Scatter updated latents back into persistent request state with
+   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).

-`StepBatchSamplingParamsKey` protects the batched tensor contract. Requests can
-have different total step counts and current step indices, but shape-sensitive,
-CFG-sensitive, and LoRA fields must be compatible. FIFO head-of-line blocking
-also applies here.
+This keeps the shared work limited to the denoise forward pass while preserving
+request-local scheduler state and outputs.

-Chunk-capable pipelines may emit intermediate results. Final-only step
-pipelines emit when the request finishes even if `streaming_output=True`.
+## Engine

-## Model Author Guidelines
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
+the background loop and async add-request path needed for multiple requests to
+accumulate in the scheduler.

-When converting a request-level pipeline:
+When `step_execution=True`, the engine routes work through the step-wise
+executor path. The continuous batching behavior is defined by scheduler-side
+compatibility gating and runner-side `StepInputBatch` packing.

- Reuse the same helpers as `forward()` to avoid behavior drift.
- Copy request-scoped scheduler state into `state.scheduler`.
- Advance `state.step_index` only in `step_scheduler()`.
- Keep model forward work in `denoise_step()`.
- Keep latent mutation in `step_scheduler()`.
- Keep final decoding equivalent to the tail of `forward()`.
- Store masks, condition latents, and other request-local tensors in the
-  request state rather than on the shared pipeline.
+## Current Limitations

-Before setting `supports_step_execution = True`, validate output parity for the
-same seed and sampling parameters, scheduler isolation across concurrent
-requests, abort cleanup, progress reporting, and all supported CFG paths.
+- Experimental feature; use `max_num_seqs=1` for the conservative single-request
+  step path.
+- Only native pipelines that already support `step_execution=True`.
+- Only homogeneous batches keyed by `StepBatchSamplingParamsKey` are supported.
+- `cache_backend`, KV transfer, and other request-mode extras are not wired
+  into the batched step-wise path yet.
+- Future work can relax the current same-shape restriction with richer
+  heterogeneous batching policies such as bucketing or padded execution for
+  different resolutions.

-Before setting `supports_request_batch = True`, validate the batch input
-contract, one-output-per-request result shape, request identity and error
-mapping, seeded concurrency, LoRA compatibility, and tensor IPC.
+## Related Files

-## Limitations
-
- Request batching requires an explicit pipeline capability when
-  `max_num_seqs>1`.
- Step execution requires the four-method stateful pipeline contract.
- Both batching paths currently require homogeneous compatibility keys.
- FIFO scheduling can reduce batching opportunities.
- `request_batch_max_wait_ms` trades first-request latency for burst
-  coalescing.
- All diffusion cache backends are currently unsupported in step mode.
- KV transfer is supported for newly admitted step requests; some other
-  request-mode extras remain unsupported.
- Step continuous batching remains experimental; use `max_num_seqs=1` for the
-  conservative step path.
-
-## Related Implementation
-
- Engine: [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
- Model contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
- Scheduler interface: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
- Request scheduler: [`vllm_omni/diffusion/sched/request_scheduler.py`](gh-file:vllm_omni/diffusion/sched/request_scheduler.py)
- Step scheduler: [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
- Runner: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
- Request batch: [`vllm_omni/diffusion/worker/request_batch.py`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
- Step input batch: [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Step scheduler:
+  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
+- Runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Input batch:
+  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)
@@ -0,0 +1,175 @@
+# Request-Level Batching for Diffusion
+
+This document describes the request-mode batching path for diffusion pipelines.
+For end-user enablement and tuning, see
+[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
+
+This is separate from
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+Request-level batching runs one full pipeline `forward()` over a static batch of
+compatible requests. Step-wise continuous batching admits work between denoise
+steps when `step_execution=True`.
+
+## Why It Helps
+
+The request-level design avoids coupling several logical requests to one request
+object. This keeps request identity, abort/error handling, and per-request
+metadata unambiguous while still allowing one fused pipeline forward pass for
+bursty or concurrent traffic.
+
+## Overview
+
+With request-level batching enabled:
+
+- each `OmniDiffusionRequest` contains one `prompt` and one `request_id`
+- the scheduler groups compatible waiting requests into one scheduler wave
+- `DiffusionRequestBatch` wraps the scheduled requests for pipeline `forward()`
+- batch-capable pipelines return `list[DiffusionOutput]`, one output per
+  request
+- `BatchRunnerOutput` maps each result back to its original `request_id`
+
+Pipelines opt in with `supports_request_batch = True` and a `forward()` method
+that accepts `DiffusionRequestBatch` and returns `list[DiffusionOutput]`.
+Pipelines that do not opt in keep the existing per-request execution path.
+
+## Enablement
+
+Request-level batching is the request-mode path, so `step_execution` must remain
+disabled. Increase `max_num_seqs` above `1` to let the scheduler keep multiple
+compatible requests active:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4
+```
+
+For bursty online ingress, `request_batch_max_wait_ms` can add a bounded
+admission wait before the first `schedule()` of a scheduler wave:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+`request_batch_max_wait_ms=0` disables this wait and is the default.
+
+## Request Contract
+
+`OmniDiffusionRequest` represents one logical request. It owns one prompt,
+sampling parameters, request id, and request-local metadata. Runtime batches are
+formed by the scheduler and represented separately from the request payload.
+
+Runtime batching is represented by:
+
+- [`DiffusionSchedulerOutput`](gh-file:vllm_omni/diffusion/sched/interface.py)
+  for scheduled request ids and request payloads
+- [`DiffusionRequestBatch`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
+  for the pipeline-facing request batch
+- [`BatchRunnerOutput`](gh-file:vllm_omni/diffusion/worker/utils.py) for
+  per-request results
+
+`DiffusionRequestBatch` intentionally exposes compatibility properties such as
+`prompts`, `sampling_params`, `request_id`, and `kv_sender_info` so migrated
+pipelines can stay close to upstream code while using a batch-aware contract.
+
+## Scheduler
+
+The scheduler derives its capacity from `max_num_seqs` through
+`max_num_running_reqs`. It exposes waiting/running queue counters so the engine
+can decide whether admission wait is useful before scheduling a new wave.
+
+Batch compatibility is controlled by
+[`RequestBatchSamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key
+contains shape-sensitive and guidance-sensitive fields, including output count
+and LoRA identity. Requests with incompatible shapes, CFG settings, output
+counts, LoRA adapters, or LoRA scales are kept in separate batches.
+
+Admission is conservative:
+
+- the scheduler only batches compatible requests
+- FIFO ordering is preserved
+- an incompatible request at the head of the waiting queue blocks later
+  compatible requests
+
+## Engine
+
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) resolves
+request-batch capability during initialization from the configured pipeline
+class, including custom pipeline classes.
+
+The capability check uses the pipeline class attribute
+`supports_request_batch = True`. Pipelines that set this attribute must implement
+a request-batch-compatible `forward()` contract and return one
+`DiffusionOutput` per request; the runner validates that return shape at runtime.
+
+When the selected pipeline is batch-capable and `step_execution=False`, request
+mode routes scheduler waves through the batch executor path. Otherwise it keeps
+the per-request executor path.
+
+The optional admission wait runs only when:
+
+- request batching is supported
+- `step_execution=False`
+- `request_batch_max_wait_ms > 0`
+- no requests are currently running
+
+The wait exits early when the waiting queue reaches capacity, when the queue is
+stable for a short window, when the deadline expires, or when the engine stops.
+
+## Executor And Runner
+
+The executor exposes two request-mode entries:
+
+- `execute_request`: one worker call per scheduled request
+- `execute_batch`: one worker call for the whole `DiffusionSchedulerOutput`
+
+On the batch path, the worker builds a `DiffusionRequestBatch` and runs the
+pipeline once. Request-local setup remains per request:
+
+- KV transfer metadata
+- random generator and seed handling
+- request output/error/abort mapping
+
+Shared batch setup happens once per batch when possible:
+
+- cache refresh
+- LoRA activation for the homogeneous adapter key
+- pipeline `forward(req_batch)`
+
+Large tensor IPC still uses the shared-memory packing path. The packer traverses
+both normal `RunnerOutput.result` wrappers and nested batch results so batched
+outputs do not fall back to pickle IPC for tensor payloads.
+
+## Current Limitations
+
+- Only pipelines that declare the request-batch contract use fused batch
+  execution.
+- Batches are homogeneous under `RequestBatchSamplingParamsKey`; heterogeneous resolution or
+  incompatible guidance settings do not co-batch yet.
+- FIFO scheduling can reduce batching opportunities when an incompatible
+  request is at the front of the queue.
+- `request_batch_max_wait_ms` improves burst coalescing but can add latency to
+  the first request in a scheduler wave. Keep it small for latency-sensitive
+  serving.
+- Step-wise continuous batching is documented separately and only applies when
+  `step_execution=True`.
+
+## Related Files
+
+- Request object and request batch:
+  [`vllm_omni/diffusion/request.py`](gh-file:vllm_omni/diffusion/request.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Engine:
+  [`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
+- Worker runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Executor interface:
+  [`vllm_omni/diffusion/executor/abstract.py`](gh-file:vllm_omni/diffusion/executor/abstract.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_engine.py`](gh-file:tests/diffusion/test_diffusion_engine.py)
@@ -0,0 +1,131 @@
+# Adding Step Execution Support for Diffusion Pipelines
+
+This guide documents vLLM-Omni's stepwise diffusion contract for model authors
+and contributors implementing `step_execution=True` support for a diffusion
+pipeline.
+
+For end-user enablement, supported models, and current limitations, see
+[Step Execution](../../user_guide/diffusion/step_execution.md).
+
+This document describes the base step-execution contract only. For the
+experimental batching policy layered on top of the step-wise path, see
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+
+## Current Support Scope
+
+`step_execution` is **not** a generic diffusion toggle. It only works for
+pipelines that implement the segmented stateful contract in
+[`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
+
+This page is intentionally author-facing. Treat runtime enablement
+(`step_execution=True` in Python or `--step-execution` in serving) as an
+opt-in user knob layered on top of the implementation contract below.
+
+Current in-tree support:
+
+| Pipeline | Example models | Step execution |
+|----------|----------------|----------------|
+| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
+| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
+
+Current engine/runtime limitations:
+
+- Continuous batching with `max_num_seqs > 1` is experimental and documented in
+  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+  Keep `max_num_seqs=1` for the conservative single-request step path.
+- `cache_backend` is not supported in step mode.
+- Request-mode extras such as KV transfer are not wired into step mode yet.
+- Unsupported pipelines now fail early during model loading instead of failing on the first request.
+
+## Execution Contract
+
+Step mode is driven by four pipeline methods plus the shared mutable request
+state object:
+
+- `prepare_encode(state)`: one-time request preparation.
+- `denoise_step(state)`: compute the noise prediction for the current step.
+- `step_scheduler(state, noise_pred)`: mutate latents and advance step state.
+- `post_decode(state)`: decode the final output after denoising is complete.
+
+The state lives in
+[`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
+as `StepRequestState`. Store request-scoped tensors there, or use
+`state.extra` for model-specific fields that do not justify extending the core
+dataclass.
+
+The worker-side step loop lives in
+[`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py):
+
+1. `prepare_encode()` runs once for a new request.
+2. `denoise_step()` runs every scheduler tick.
+3. `step_scheduler()` mutates `state.latents` and advances `state.step_index`.
+4. `post_decode()` runs exactly once after `state.denoise_completed` becomes true.
+
+## Recommended Split
+
+When converting an existing request-level `forward()` pipeline, keep the split
+strict and mechanical:
+
+| Request-level phase | Stepwise method | What belongs there |
+|---------------------|-----------------|--------------------|
+| Input validation, prompt encoding, latent init, timestep prep, per-request scheduler creation | `prepare_encode()` | Anything that should happen once per request |
+| Transformer forward / noise prediction | `denoise_step()` | Pure denoise computation for the current timestep |
+| `scheduler.step(...)` and `step_index += 1` | `step_scheduler()` | Only latent/state mutation for one step |
+| VAE decode / postprocess | `post_decode()` | Final decode only |
+
+Keep the stepwise path reusing the same helpers as the request-level path
+whenever possible. Reimplementing the denoise loop from scratch is the easiest
+way to introduce behavioral drift.
+
+## Qwen-Image Reference
+
+[`pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)
+is the reference implementation and is split correctly for the current
+contract:
+
+- `prepare_encode()` reuses `_prepare_generation_context()` so prompt encoding,
+  latent init, timestep creation, CFG setup, and shape bookkeeping stay aligned
+  with `forward()`.
+- `prepare_encode()` deep-copies `self.scheduler` **after**
+  `prepare_timesteps()` so request-specific scheduler state is isolated.
+- `denoise_step()` reuses `_build_denoise_kwargs()` plus
+  `predict_noise_maybe_with_cfg()`, so sequential CFG, CFG-parallel, and
+  non-CFG behavior stay identical to the request-level path.
+- `step_scheduler()` only calls
+  `scheduler_step_maybe_with_cfg(..., per_request_scheduler=state.scheduler)`
+  and increments `state.step_index`.
+- `post_decode()` reuses `_decode_latents()`, so the final image decode matches
+  the normal `forward()` path.
+
+That decomposition is the target pattern for future models.
+
+## Rules For New Pipelines
+
+- Do not keep request-scoped scheduler state on `self.scheduler`. Copy it into
+  `state.scheduler` during `prepare_encode()`.
+- Do not mutate `state.step_index` inside `denoise_step()`. Only
+  `step_scheduler()` should advance the step.
+- Do not decode partial outputs in `denoise_step()` or `step_scheduler()`.
+- If the request-level pipeline has condition latents, masks, or edit-specific
+  tensors, store them in `state` or `state.extra`, not in global pipeline
+  attributes.
+- Preserve CFG behavior by sharing the same helper path used by `forward()`.
+- Keep `post_decode()` equivalent to the tail of `forward()`.
+
+## Validation Checklist
+
+Before marking a pipeline as `supports_step_execution = True`, verify:
+
+- Stepwise output matches request-level output for the same seed and sampling params.
+- Per-request scheduler state is isolated across concurrent requests.
+- Abort during denoise does not leak cached state.
+- `step_index` reported by `RunnerOutput` matches the scheduler progress.
+- CFG-parallel and non-CFG paths both work if the request-level pipeline supports them.
+
+## Related Files
+
+- Contract: [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py)
+- State: [`vllm_omni/diffusion/worker/utils.py`](gh-file:vllm_omni/diffusion/worker/utils.py)
+- Runner loop: [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Scheduler transport: [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Reference pipeline: [`vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py`](gh-file:vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py)
@@ -1,386 +0,0 @@
-# Async Omni Output Materialization
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Performance](#performance)
-3. [Architecture](#architecture)
-4. [Enabling the Feature](#enabling-the-feature)
-5. [Compatibility and Fallbacks](#compatibility-and-fallbacks)
-6. [Related Files](#related-files)
-
-## Overview
-
-Async Omni output materialization moves CPU-side construction of
-`OmniModelRunnerOutput` out of the autoregressive (AR) decode critical path.
-It complements [Async Chunk](async_chunk.md): async chunk pipelines partial
-outputs between stages, while async output materialization prevents the
-construction of those partial outputs from blocking the next decode step.
-
-An AR stage must return sampled token IDs quickly because the scheduler needs
-them for the next decode step. However, an Omni stage can also produce hidden
-states, multimodal tensors, streaming inter-stage payloads, and connector
-metadata. Building these payloads can require device-to-host (D2H) copies,
-tensor slicing, flattening, and Python object construction.
-
-Without this feature, all of that work runs inline in `sample_tokens()`:
-
-```text
-sample tokens
-  -> update decode state
-  -> copy hidden and multimodal outputs to CPU
-  -> build per-request payloads
-  -> build the streaming wire payload
-  -> collect connector signals
-  -> return sampled tokens
-  -> launch the next decode step
-```
-
-With async output materialization, only work required by the next decode step
-stays on the critical path:
-
-```text
-sample tokens
-  -> update decode state
-  -> snapshot output state and start asynchronous D2H copies
-  -> register sampled tokens
-  -> launch the next decode step
-
-background output path
-  -> wait for the payload snapshot
-  -> build per-request payloads
-  -> build the streaming wire payload
-  -> collect connector signals
-  -> construct OmniModelRunnerOutput
-```
-
-This allows payload construction for step `N` to overlap with GPU decode work
-for step `N + 1`. It does not change model computation or generated output.
-
-!!! note "Platform and validation scope"
-    The performance and correctness validation in this document covers CUDA
-    and ROCm. This is a validation scope, not a runtime platform guard: XPU
-    inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so those
-    backends may enter the asynchronous path when its other guards pass. XPU
-    and MUSA have not been validated for this optimization. Ascend NPU uses a
-    separate `NPUARModelRunner` path that still constructs
-    `OmniModelRunnerOutput` and drains connector output synchronously before
-    wrapping the result as an asynchronous runner output.
-
-<p align="center">
-  <img
-    alt="Decode step gap before and after async output materialization"
-    src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"
-    width="100%"
-  >
-</p>
-
-<p align="center">
-  <em>
-    Figure 1: Moving payload construction off the decode path reduces the
-    observed gap between consecutive Talker steps from about 2.8 ms to 41 µs.
-    Adapted from the
-    <a href="https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization">
-      Qwen3-Omni optimization blog
-    </a>.
-  </em>
-</p>
-
-For Qwen3-Omni, the feature is enabled for both AR stages:
-
- **Thinker**: Defers hidden-state and multimodal payload materialization.
- **Talker**: Runs its decode-state postprocess eagerly, then defers codec
-  payload materialization. Because Code2Wav consumes codec codes rather than
-  Talker hidden states, the Talker also skips unnecessary hidden-state D2H.
- **Code2Wav**: Does not use this path because it is a generation stage rather
-  than an AR stage.
-
-Qwen3-TTS uses the same mechanism in its AR Talker stage. The Talker updates
-the decode state it needs for the next step before returning, then constructs
-the codec payload for Code2Wav in the background.
-
-## Performance
-
-In the controlled Qwen3-Omni optimization sweep, enabling async output
-materialization on top of CUDA Graph and async chunk produced the following
-results at concurrency 64:
-
-| Configuration | Request throughput | Mean audio TTFP | Mean audio RTF |
-|---|---:|---:|---:|
-| CUDA Graph + async chunk | 9.3 req/s | 655 ms | 0.63 |
-| + async output materialization | 11.3 req/s | 631 ms | 0.47 |
-| Change | **+22%** | **-4%** | **-25%** |
-
-The sweep used `Qwen/Qwen3-Omni-30B-A3B-Instruct`, 640 Seed-TTS English
-prompts, and one replica each for Thinker, Talker, and Code2Wav. Async output
-materialization recovers throughput by removing CPU payload construction from
-the decode critical path without regressing time to first audio packet (TTFP).
-See the
-[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)
-for the complete benchmark methodology and results.
-
-The implementation was also validated against the synchronous safe-copy path.
-The text and audio output hashes matched, confirming that overlap does not
-change the generated result. See
-[PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476) for the
-correctness and profiling details.
-
-## Architecture
-
-### Output Lifecycle
-
-`OmniAsyncGPUModelRunnerOutput` extends vLLM's
-`AsyncGPUModelRunnerOutput`. It preserves the existing asynchronous
-sampled-token feedback while adding a background builder for the complete Omni
-output.
-
-The output lifecycle is:
-
-1. `GPUARModelRunner.sample_tokens()` samples tokens and performs the
-   bookkeeping required by the next decode step.
-2. The runner snapshots step-local metadata such as request IDs, token spans,
-   scheduler output, and query start locations.
-3. CUDA payload tensors are cloned before reusable model or CUDA Graph buffers
-   can be overwritten.
-4. The cloned payload is copied to pinned CPU memory on a dedicated CUDA
-   stream. A CUDA event records when the copy is ready.
-5. `OmniAsyncGPUModelRunnerOutput` starts a background thread that waits for
-   the payload event and builds `OmniModelRunnerOutput`.
-6. The AR runner immediately registers the sampled-token CPU copy with the
-   input batch and returns, allowing async scheduling to advance.
-7. When the engine calls `get_output()`, it joins the background builder,
-   propagates any builder exception, and finalizes sampled tokens and logprobs
-   through the upstream output implementation.
-
-```text
-GPU / decode path                         Background output path
-
-forward + sample
-      |
-      +-- clone output tensors
-      +-- enqueue D2H copy --------------> wait for D2H event
-      +-- register sampled tokens               |
-      +-- return async output                    +-- slice per request
-      |                                         +-- build multimodal payloads
-next decode step                                +-- partition streaming payloads
-                                                +-- build the wire payload
-                                                +-- drain connector signals
-                                                +-- build OmniModelRunnerOutput
-```
-
-### Safe State Snapshots
-
-The next scheduler step can mutate runner state while the background builder is
-still active. Most step-local builder inputs are therefore detached snapshots.
-The snapshotted state includes:
-
- scheduler token counts and speculative-token metadata
- request IDs and request-to-batch-index mappings
- sampled token IDs, logprobs, and prompt logprobs
- query start locations and scheduled-token spans
- hidden states and multimodal output tensors
- KV-connector and encoder-cache outputs captured for the step
-
-CUDA tensor cloning is required because CUDA Graph and model output buffers can
-be reused by the next forward pass. The dedicated copy stream and pinned host
-buffers make the D2H transfer asynchronous; the snapshot retains its cloned
-CUDA sources until the transfer event completes.
-
-### Live Runner State and Ownership
-
-The complete builder input is not a detached snapshot. Connector output state
-remains runner-owned: `get_omni_connector_output()` drains live, per-cycle
-connector signals after the streaming payload is built. Correctness therefore
-depends on that builder being the only consumer that drains connector signals
-for its output cycle. Receive-side connector state written by the background
-receiver is coordinated by the connector mixin's `_lock`.
-`OmniAsyncGPUModelRunnerOutput.get_output()` joins the builder and is the
-completion and exception boundary before the materialized output is consumed.
-
-The shared output builder also contains a full-payload accumulation branch
-that reads `self.requests`. That branch is not reachable when async Omni output
-materialization is enabled: this feature requires `async_chunk`, while
-`should_accumulate_full_payload_output()` returns `False` whenever
-`async_chunk` is enabled. The feature path partitions each step's payload into
-streaming inter-stage and client handoffs instead.
-
-### Output Builder
-
-The background builder performs the work that previously ran inline:
-
- resolves which requests need downstream payloads
- converts hidden and multimodal tensors into per-request CPU payloads
- applies model-specific payload processing
- partitions per-step payloads into inter-stage and client streams
- creates the tensor-only `multimodal_outputs` wire payload
- drains live connector readiness signals after building the streaming payload
- constructs the final `OmniModelRunnerOutput`
-
-Input-side connector operations remain synchronous with model execution. In
-particular, receiving inputs and flushing previously completed connector
-outputs are not deferred because they affect scheduler-visible request state.
-
-### Qwen3-Omni Stage Behavior
-
-| Stage | Async output behavior | Reason |
-|---|---|---|
-| Thinker | Snapshots hidden states and multimodal outputs; builds the downstream payload in the background | The Talker needs the payload, but the next Thinker decode step only needs sampled-token feedback |
-| Talker | Runs lightweight postprocess eagerly; snapshots codec outputs; omits hidden states from the downstream payload | `hidden_states.last` is needed by the next Talker step, while Code2Wav only needs codec codes |
-| Code2Wav | Uses the normal generation-stage output path | Code2Wav is not executed by `GPUARModelRunner` |
-
-The Qwen3-TTS Talker follows the same pattern as the Qwen3-Omni Talker:
-postprocess runs eagerly because the next decode step needs its updated state,
-while codec payload construction runs asynchronously. Its Code2Wav stage also
-uses the normal generation-stage path.
-
-## Enabling the Feature
-
-There is no separate `async_omni_output` command-line or YAML option. The
-feature is selected automatically for model stages that opt in when the runtime
-conditions are safe. On the validated CUDA and ROCm configurations, using the
-bundled deployment profile is enough to turn it on for the supported
-Qwen3-Omni and Qwen3-TTS pipelines. The selection logic has no CUDA/ROCm-only
-guard; see [Compatibility and Fallbacks](#compatibility-and-fallbacks) for the
-other platform paths. The same profile does not enable background Omni
-materialization on Ascend NPU.
-
-### Example 1: Qwen3-Omni
-
-Start the full Qwen3-Omni pipeline:
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --omni \
-  --port 8091
-```
-
-The model registry automatically loads
-`vllm_omni/deploy/qwen3_omni_moe.yaml`. Its relevant settings are:
-
-```yaml
-async_chunk: true
-
-stages:
-  - stage_id: 0  # Thinker
-    enable_prefix_caching: false
-    # async_scheduling defaults to true for AR stages
-
-  - stage_id: 1  # Talker
-    enable_prefix_caching: false
-    # async_scheduling defaults to true for AR stages
-
-  - stage_id: 2  # Code2Wav
-    enable_prefix_caching: false
-    async_scheduling: false
-```
-
-With this profile, async output materialization activates automatically for:
-
- **Stage 0, Thinker**: Defers hidden-state and multimodal payload
-  construction.
- **Stage 1, Talker**: Defers codec payload construction after eagerly
-  updating its decode state.
-
-Stage 2 does not use this feature because Code2Wav is not an AR stage.
-
-### Example 2: Qwen3-TTS
-
-Start Qwen3-TTS with any checkpoint that uses the Qwen3-TTS Talker pipeline.
-For example:
-
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-  --omni \
-  --port 8091
-```
-
-The model registry automatically loads `vllm_omni/deploy/qwen3_tts.yaml`. Its
-relevant settings are:
-
-```yaml
-async_chunk: true
-
-stages:
-  - stage_id: 0  # Talker
-    async_scheduling: true
-    enable_prefix_caching: false
-
-  - stage_id: 1  # Code2Wav
-    enable_prefix_caching: false
-```
-
-Async output materialization activates only for Stage 0, the AR Talker. The
-Talker keeps the state required by its next decode step on the GPU, skips an
-unnecessary hidden-state D2H copy, and builds the codec payload for Code2Wav in
-the background. Stage 1 uses the generation-stage output path.
-
-The same behavior applies to the Base and VoiceDesign Qwen3-TTS checkpoints
-because they use the same Talker implementation.
-
-!!! warning
-    Do not pass `--no-async-chunk` or enable prefix caching when you want this
-    optimization. Either change causes the runner to fall back to synchronous
-    output construction. Enabling async chunk alone cannot activate the feature
-    for a model that has not opted into async Omni output.
-
-!!! note "Prefix cache compatibility"
-    Async Omni output materialization and Omni prefix caching cannot currently
-    run together. `_should_use_async_omni_output()` returns `False` whenever
-    `self.omni_prefix_cache` is present, so prefix caching selects synchronous
-    output materialization. Supporting both features requires snapshotting or
-    otherwise synchronizing prefix-cache merge and update state before the
-    background output path can consume it safely.
-
-## Compatibility and Fallbacks
-
-`GPUARModelRunner` uses the async output path only when all of the following
-runtime conditions hold:
-
-| Requirement | Reason |
-|---|---|
-| AR async scheduling is enabled | The optimization relies on the scheduler advancing while the prior output is materialized |
-| `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |
-| The model stage opts in with `use_async_omni_output` | Models must declare that their output lifecycle is safe to defer |
-| Omni prefix cache is disabled | Prefix-cache merge and update ordering currently requires synchronous materialization |
-| Speculative decoding is disabled | Speculative output state is not included in this deferred path |
-| Routed-expert output is disabled | Routed-expert extraction currently requires the synchronous path |
-| Postprocess is absent or explicitly runs eagerly | State needed by the next decode step must be updated before the runner returns |
-
-These checks are evaluated per stage on runners based on `GPUARModelRunner`.
-An unsupported combination does not prevent serving; it only falls back to
-synchronous output construction for that stage. CUDA and ROCm are the validated
-platforms. XPU inherits `GPUARModelRunner`, and MUSA selects `GPUARWorker`, so
-they may select the async path when these checks pass, but this optimization is
-not yet validated on either backend.
-
-Ascend NPU does not use this selection path. `NPUARModelRunner.sample_tokens()`
-fully constructs `OmniModelRunnerOutput`, calls
-`get_omni_connector_output()`, and only then creates the asynchronous wrapper,
-so Omni payload materialization remains synchronous.
-
-!!! note
-    `use_async_omni_output`,
-    `eager_omni_postprocess_before_async_output`, and
-    `omni_pooler_payload_include_hidden` are model implementation contracts,
-    not user-facing configuration fields.
-
-## Related Files
-
- `vllm_omni/worker/gpu_ar_model_runner.py`: Async output object, safe tensor
-  snapshots, compatibility guards, and deferred Omni output builder.
- `vllm_omni/platforms/npu/worker/npu_ar_model_runner.py`: Separate Ascend NPU
-  runner, where Omni output materialization remains synchronous.
- `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`: Thinker and
-  Talker opt-in behavior.
- `vllm_omni/deploy/qwen3_omni_moe.yaml`: Default Qwen3-Omni deployment
-  settings.
- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py`: Qwen3-TTS
-  Talker opt-in and eager postprocess behavior.
- `vllm_omni/deploy/qwen3_tts.yaml`: Default Qwen3-TTS deployment settings.
- `tests/worker/test_gpu_ar_model_runner.py`: Snapshot, guard, connector
-  ordering, and background error-propagation tests.
- [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.
- [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):
-  Optimization context and controlled performance results.
- [PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476): Feature
-  implementation, validation, and review history.
@@ -264,7 +264,7 @@ Complete examples in the codebase:
 | **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Standard TP | Full implementation with validation |
 | **FLUX** | `vllm_omni/diffusion/models/flux/flux_transformer.py` | Dual-stream | Image + text streams |
 | **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Standard TP | With RoPE |
-| **TP Tests** | `tests/diffusion/distributed/test_tensor_parallel.py` | E2E testing | TP correctness and performance |
+| **TP Tests** | `tests/e2e/offline_inference/test_zimage_parallelism.py` | E2E testing | TP correctness and performance |
 | **Constraint Tests** | `tests/diffusion/models/z_image/test_zimage_tp_constraints.py` | Unit testing | Validation logic |

 ---
@@ -1,8 +1,6 @@
 # Design Documents

-This section contains design documents and architecture specifications for
-vLLM-Omni. The sidebar groups documents by the system concern they describe,
-while the source paths remain stable so existing links continue to work.
+This section contains design documents and architecture specifications for vLLM-Omni.

 ## Architecture Documents

@@ -10,63 +8,18 @@ while the source paths remain stable so existing links continue to work.

 ## Feature Design Documents

-### Runtime and stage execution
-
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
- [Async Chunk](feature/async_chunk.md)
- [Async Diffusion Output](feature/async_diffusion_output.md)
- [Async Omni Output Materialization](feature/omni_async_output_materialization.md)
- [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
+- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
+- [Request-Level Batching for Diffusion](feature/diffusion_request_level_batching.md)
+- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)

-### Communication
-
-#### OmniConnector implementations
-
- [Mooncake Store Connector](feature/omni_connectors/mooncake_store_connector.md)
- [Mooncake Transfer Engine Connector](feature/omni_connectors/mooncake_transfer_engine_connector.md)
- [Mori Transfer Engine Connector](feature/omni_connectors/mori_transfer_engine_connector.md)
- [Shared Memory Connector](feature/omni_connectors/shared_memory_connector.md)
- [Yuanrong Store Connector](feature/omni_connectors/yuanrong_connector.md)
- [Yuanrong Transfer Engine Connector](feature/omni_connectors/yuanrong_transfer_engine_connector.md)
-
-### Diffusion acceleration
-
-#### Parallelism
-
- [CFG-Parallel](feature/cfg_parallel.md)
- [Expert Parallel](feature/expert_parallel.md)
- [Hybrid Sharded Data Parallel (HSDP)](feature/hsdp.md)
- [Pipeline Parallel](feature/pipeline_parallel.md)
- [Sequence Parallel](feature/sequence_parallel.md)
- [Tensor Parallel](feature/tensor_parallel.md)
- [VAE Patch Parallelism](feature/vae_parallel.md)
-
-#### Attention Backends
-
- [Skip-Softmax](feature/skip_softmax.md)
-
-#### Quantization
-
- [Quantization Overview](../user_guide/quantization/overview.md)
-
- [Cache-DiT](feature/cache_dit.md)
- [TeaCache](feature/teacache.md)
- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)
-
-## Infrastructure and Performance
+## Infrastructure Design Documents

 - [Prometheus Metrics](metrics.md)
- [Speech Generation Performance Optimizations](qwen3_omni_tts_performance_optimization.md)

 ## Module Design Documents

-### Runtime Modules
-
 - [AR Module](module/ar_module.md)
 - [DIT Module](module/dit_module.md)
-
-### Orchestration
-
 - [Entrypoint Module](module/entrypoint_module.md)
- [AsyncOmni Architecture (Qwen3-Omni Example)](module/async_omni_architecture.md)
@@ -148,13 +148,7 @@ Upstream vLLM's `Scheduler.make_stats()` runs on every AR generation step, retur

 `OmniSchedulerMixin.make_stats()` (in `vllm_omni/core/sched/omni_scheduler_mixin.py`) throttles stats emission to at most once per second **per scheduler** — i.e. per `(stage, replica)` since each replica owns its own scheduler instance. Between intervals it returns `None`, which the engine core skips serializing. This keeps gauges fresh enough for Prometheus scrapes (typically 15-30s intervals) while eliminating the per-step overhead.

-The orchestrator side does not add its own throttle on top. Scheduler gauges are
-recorded when `raw_outputs.scheduler_stats is not None` (i.e. this replica's
-scheduler passed its own 1 Hz gate), while output-batch request/token metrics
-are recorded from `IterationStats` whenever a non-empty output batch is
-processed. A previous global `_last_stats_ts` on the orchestrator was removed
-because it starved every replica other than the first to emit within each
-second.
+The orchestrator side does not add its own throttle on top: the per-replica recording loop gates only on `raw_outputs.scheduler_stats is not None` (i.e. this replica's scheduler passed its own 1 Hz gate). A previous global `_last_stats_ts` on the orchestrator was removed because it starved every replica other than the first to emit within each second.

 ## Metric Definitions

@@ -252,7 +252,8 @@ while True:
  batch of compatible independent requests for one full pipeline forward when
  the pipeline supports request-level batching; the step-wise path can also
  admit or remove compatible requests between denoise steps. See
-  [Diffusion Continuous Batching](../feature/diffusion_continuous_batching.md).
+  [Request-Level Batching](../feature/diffusion_request_level_batching.md) and
+  [Continuous Batching for Step-Wise Diffusion](../feature/diffusion_continuous_batching.md).

 ---

@@ -419,7 +419,8 @@ Notes:
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
  --omni \
-  --port 8091
+  --port 8091 \
+  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
 ```

 #### 3) Key config knobs
@@ -493,7 +494,7 @@ The default config (`qwen3_tts.yaml`) enables the full optimization stack:
 vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
  --omni \
  --port 8000 \
-  --no-async-chunk
+  --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_no_async_chunk.yaml
 ```

 #### 3) Key config knobs
@@ -1,8 +1,5 @@
 # Installation

-!!! important
-    vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.
-
 vLLM-Omni supports the following hardware platforms:

 - [GPU](gpu.md)
@@ -5,7 +5,7 @@
 # --8<-- [end:requirements]
 # --8<-- [start:set-up-using-python]

-vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
+vLLM-Omni depends vLLM. So please follow instructions below mainly for vLLM.

 !!! note
    PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
@@ -124,7 +124,7 @@ If you want to specify the base vLLM version:
 ```bash
 DOCKER_BUILDKIT=1 docker build \
  -f docker/Dockerfile.cuda \
-  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0 \
+  --build-arg BASE_IMAGE=vllm/vllm-openai:v0.22.1 \
  -t vllm-omni-cuda .
 ```

@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built

 ```bash
 # vLLM-Ascend image
-# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
-# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
+# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.25.0
+# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.25.0-a3
 docker run --rm \
    --name vllm-omni-npu \
    --shm-size=1g \
@@ -28,11 +28,11 @@ docker run --rm \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /root/.cache:/root/.cache \
    -p 8000:8000 \
-    -it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
+    -it quay.io/atlas-ci/vllm-ascend:0.25.0-a3 bash

 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.25.0 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
@@ -53,20 +53,16 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
 You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)

 ```bash
-# Pin vLLM and vLLM-Ascend to the v0.26 release line
-git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
-cd vllm
+# Pin vLLM version to 0.18.0
+git clone -b v0.18.0 https://github.com/vllm-project/vllm.git
 VLLM_TARGET_DEVICE=empty pip install -v -e .
-cd ..

-git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
-cd vllm-ascend
+git clone -b v0.18.0rc1 https://github.com/vllm-project/vllm-ascend.git
 pip install -v -e .
-cd ..

 # Install vLLM-Omni from the latest main branch
 git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
+cd /vllm-workspace/vllm-omni
 pip install -v -e . --no-build-isolation
 # or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -88,8 +84,8 @@ Supported images as following.
 Here's an example deployment command that has been verified on 4 x NPUs:

 ```bash
-# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
-# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
+# Atlas A2: quay.io/ascend/vllm-omni:v0.25.0
+# Atlas A3: quay.io/ascend/vllm-omni:v0.25.0-a3
 docker run --rm \
    --name vllm-omni-a3 \
    --shm-size=4g \
@@ -114,7 +110,7 @@ docker run --rm \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v ~/.cache:/root/.cache \
    -p 8091:8091 \
-    -it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
+    -it quay.io/ascend/vllm-omni:v0.25.0-a3 bash
 ```

 !!! tip
@@ -126,9 +122,9 @@ Or build IMAGE from **source code**:
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 # A2
-# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
+docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu .
 # A3
-docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
+# docker build -t vllm-omni-dev-image:latest -f ./docker/Dockerfile.npu.a3 .
 ```

 # --8<-- [end:pre-built-images]
@@ -87,8 +87,9 @@ if __name__ == "__main__":

 !!! info

-    For diffusion request batching, step execution, and streaming controls, see
-    [Diffusion Execution Modes](../user_guide/diffusion/execution_modes.md).
+    For diffusion request-level batching controls such as `max_num_seqs` and
+    `request_batch_max_wait_ms`, see
+    [Request-Level Batching](../user_guide/diffusion/request_batching.md).

 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)

@@ -44,7 +44,6 @@ th {
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
 | `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
-| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
 | `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
@@ -276,42 +276,20 @@ Client -> Server:

 | Message | Description |
 |---------|-------------|
-| `{"type": "session.config", ...}` | Session configuration (first message; may be resent between utterances to change it) |
+| `{"type": "session.config", ...}` | Session configuration (sent once, first message) |
 | `{"type": "input.text", "text": "..."}` | Text chunk |
-| `{"type": "input.done"}` | End of utterance: flushes the buffer and keeps the connection open |
-| `{"type": "session.close"}` | End of connection |
+| `{"type": "input.done"}` | End of input, flushes remaining buffer |

 Server -> Client:

 | Message | Description |
 |---------|-------------|
-| `{"type": "audio.start", "utterance_index": 0, "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
+| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
 | Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
-| `{"type": "audio.done", "utterance_index": 0, "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
-| `{"type": "session.done", "utterance_index": 0, "total_sentences": N}` | Flushed utterance complete |
+| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
+| `{"type": "session.done", "total_sentences": N}` | Session complete |
 | `{"type": "error", "message": "..."}` | Non-fatal error |

-### Flushing vs. Closing
-
-`input.done` is a flush, not a disconnect. The server synthesizes the buffered
-text, emits `session.done`, and then waits on the same connection for the next
-utterance, so a client that speaks repeatedly (for example one driven by an
-upstream LLM) pays the WebSocket handshake once instead of once per utterance.
-
-* The session config is sticky. Send `input.text` again straight after
-  `session.done` to reuse it, or send another `session.config` first to change
-  voice, format, or reference audio. A `session.config` sent while text is
-  still buffered is rejected so no pending input is silently dropped.
-* An utterance is the flush unit, not a linguistic one: it is whatever text was
-  buffered when `input.done` arrived, of any length, synthesized as one request.
-  `utterance_index` counts those flushes across the connection, so it tells you
-  which `input.done` a frame belongs to. `sentence_index` counts within one
-  flush and so pairs with `total_sentences`, which means every utterance reports
-  `sentence_index: 0` of `total_sentences: 1` (or `0` for an empty buffer).
-* End the connection with `session.close`, or by closing the socket. An idle
-  connection is still closed after the server's idle timeout, which now also
-  applies to the gap between utterances.
-
 ### Session Config Parameters

 All REST API parameters are supported, plus:
@@ -1,150 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="1024" height="600" viewBox="0 0 1024 600" role="img" aria-labelledby="title desc">
-  <title id="title">Qwen3-Omni Talker step gap before and after asynchronous output materialization</title>
-  <desc id="desc">Before the optimization, synchronous D2H copy, per-request payload construction, streaming payload handoff, and connector-signal drain separate consecutive Talker forward and sample steps by about 2.8 milliseconds. After the optimization, that Omni output work runs on a background builder while the next Talker step executes, leaving a boundary of about 41 microseconds for snapshot, D2H enqueue, state update, and sampled-token feedback.</desc>
-  <defs>
-    <marker id="time-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#64748b"/>
-    </marker>
-    <marker id="red-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="5" markerHeight="5" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#b91c1c"/>
-    </marker>
-    <marker id="red-start" viewBox="0 0 10 10" refX="2" refY="5" markerWidth="5" markerHeight="5" orient="auto">
-      <path d="M 10 0 L 0 5 L 10 10 z" fill="#b91c1c"/>
-    </marker>
-    <marker id="blue-end" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto">
-      <path d="M 0 0 L 10 5 L 0 10 z" fill="#2563eb"/>
-    </marker>
-    <style>
-      .canvas { fill: #ffffff; stroke: #e5e7eb; }
-      .title { font: 700 22px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
-      .subtitle { font: 500 13px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .phase { font: 700 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1f2937; }
-      .phase-sub { font: 500 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .lane { font: 600 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #374151; }
-      .block-title { font: 700 10px system-ui, -apple-system, Segoe UI, sans-serif; letter-spacing: .25px; fill: #ffffff; }
-      .block-op { font: 500 9.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
-      .axis { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #94a3b8; }
-      .measure { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #b91c1c; }
-      .inset-title { font: 700 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
-      .inset-step { font: 600 10px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; }
-      .note-title { font: 700 12px system-ui, -apple-system, Segoe UI, sans-serif; fill: #1e3a8a; }
-      .note-text { font: 500 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; }
-      .stat-cap { font: 600 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .stat-big { font: 800 16px system-ui, -apple-system, Segoe UI, sans-serif; fill: #111827; }
-      .stat-sub { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #64748b; }
-      .before-card { fill: #eef4ff; stroke: #c7dbf6; }
-      .after-card { fill: #eafaf0; stroke: #bfe7cf; }
-      .talker { fill: #16a34a; }
-      .sync-output { fill: #ea580c; }
-      .async-output { fill: #2563eb; }
-      .zoom-card { fill: #fff7f7; stroke: #f3caca; }
-      .note-card { fill: #eff6ff; stroke: #bfdbfe; }
-      .stat-blue { fill: #eef4ff; stroke: #c7dbf6; }
-      .stat-green { fill: #eafaf0; stroke: #bfe7cf; }
-      .stat-gray { fill: #f4f5f7; stroke: #e2e5ea; }
-      .guide { stroke: #e2e8f0; stroke-width: 1; }
-      .time-line { stroke: #64748b; stroke-width: 1.3; fill: none; marker-end: url(#time-arrow); }
-      .measure-line { stroke: #b91c1c; stroke-width: 1.4; marker-start: url(#red-start); marker-end: url(#red-end); }
-      .zoom-lead { stroke: #dc2626; stroke-width: 1.15; stroke-dasharray: 4 3; fill: none; }
-      .handoff { stroke: #2563eb; stroke-width: 1.3; stroke-dasharray: 3 2; fill: none; marker-end: url(#blue-end); }
-    </style>
-  </defs>
-
-  <rect x="14" y="14" width="996" height="572" rx="14" class="canvas"/>
-
-  <text x="44" y="52" class="title">Async output: what moves off the Talker decode path</text>
-  <text x="44" y="74" class="subtitle">Qwen3-Omni Talker stage &#183; consecutive decode steps in a fixed profiler window</text>
-
-  <!-- ===================== BEFORE ===================== -->
-  <rect x="44" y="94" width="144" height="110" rx="10" class="before-card"/>
-  <text x="60" y="120" class="phase">Before</text>
-  <text x="60" y="139" class="phase-sub">Omni output built</text>
-  <text x="60" y="156" class="phase-sub">synchronously on the</text>
-  <text x="60" y="173" class="phase-sub">AR critical path</text>
-
-  <text x="206" y="122" class="lane">AR path</text>
-  <line x1="300" y1="88" x2="300" y2="210" class="guide"/>
-
-  <rect x="300" y="104" width="190" height="54" rx="5" class="talker"/>
-  <text x="395" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
-  <text x="395" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <rect x="490" y="104" width="250" height="54" rx="5" class="sync-output"/>
-  <text x="615" y="120" text-anchor="middle" class="block-title">SYNC OMNI OUTPUT &#183; STEP N</text>
-  <text x="615" y="138" text-anchor="middle" class="block-op">D2H &#8594; per-request slice / payload build</text>
-  <text x="615" y="152" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
-
-  <rect x="740" y="104" width="220" height="54" rx="5" class="talker"/>
-  <text x="850" y="124" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
-  <text x="850" y="145" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <line x1="490" y1="174" x2="740" y2="174" class="measure-line"/>
-  <text x="615" y="194" text-anchor="middle" class="measure">~2.8 ms GPU gap: output work is inline</text>
-
-  <!-- ===================== AFTER ===================== -->
-  <rect x="44" y="218" width="144" height="140" rx="10" class="after-card"/>
-  <text x="60" y="244" class="phase">After</text>
-  <text x="60" y="263" class="phase-sub">Only step feedback</text>
-  <text x="60" y="280" class="phase-sub">stays inline; payload</text>
-  <text x="60" y="297" class="phase-sub">work runs in the</text>
-  <text x="60" y="314" class="phase-sub">background</text>
-
-  <text x="206" y="254" class="lane">Talker GPU</text>
-  <text x="206" y="326" class="lane">Output builder</text>
-  <line x1="300" y1="212" x2="300" y2="370" class="guide"/>
-
-  <rect x="300" y="226" width="235" height="54" rx="5" class="talker"/>
-  <text x="418" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N</text>
-  <text x="418" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <rect x="540" y="226" width="235" height="54" rx="5" class="talker"/>
-  <text x="658" y="246" text-anchor="middle" class="block-title">TALKER STAGE &#183; STEP N+1</text>
-  <text x="658" y="267" text-anchor="middle" class="block-op">forward &#8594; sample</text>
-
-  <path d="M 535 280 L 548 298" class="handoff"/>
-  <rect x="540" y="298" width="360" height="58" rx="5" class="async-output"/>
-  <text x="720" y="315" text-anchor="middle" class="block-title">BACKGROUND OMNI OUTPUT &#183; STEP N</text>
-  <text x="720" y="334" text-anchor="middle" class="block-op">wait for D2H &#8594; per-request slice / payload build</text>
-  <text x="720" y="349" text-anchor="middle" class="block-op">streaming handoff &#8594; connector-signal drain</text>
-
-  <line x1="300" y1="366" x2="930" y2="366" class="time-line"/>
-  <text x="922" y="382" text-anchor="end" class="axis">time</text>
-
-  <!-- leads from the real 41 us boundary to the magnified inset -->
-  <line x1="535" y1="280" x2="485" y2="424" class="zoom-lead"/>
-  <line x1="540" y1="280" x2="525" y2="424" class="zoom-lead"/>
-
-  <!-- ===================== 41 µs BOUNDARY ===================== -->
-  <rect x="300" y="386" width="400" height="92" rx="10" class="zoom-card"/>
-  <text x="316" y="404" class="inset-title">Zoom &#183; Talker step boundary (magnified)</text>
-
-  <rect x="320" y="416" width="165" height="26" rx="4" class="talker"/>
-  <rect x="525" y="416" width="155" height="26" rx="4" class="talker"/>
-  <text x="403" y="433" text-anchor="middle" class="inset-step">Talker step N</text>
-  <text x="603" y="433" text-anchor="middle" class="inset-step">Talker step N+1</text>
-
-  <line x1="485" y1="452" x2="525" y2="452" class="measure-line"/>
-  <text x="505" y="469" text-anchor="middle" class="measure">~41 &#181;s</text>
-
-  <rect x="720" y="386" width="240" height="92" rx="10" class="note-card"/>
-  <text x="736" y="405" class="note-title">Inline at the ~41 &#181;s boundary</text>
-  <text x="736" y="424" class="note-text">&#8226; eager Talker state update</text>
-  <text x="736" y="441" class="note-text">&#8226; output snapshot + D2H enqueue</text>
-  <text x="736" y="458" class="note-text">&#8226; sampled-token feedback</text>
-
-  <!-- ===================== SUMMARY ===================== -->
-  <rect x="44" y="494" width="296" height="70" rx="9" class="stat-blue"/>
-  <text x="60" y="514" class="stat-cap">Step-to-step gap</text>
-  <text x="60" y="538" class="stat-big">2.8 ms &#8594; 41 &#181;s</text>
-  <text x="60" y="555" class="stat-sub">~68&#215; tighter</text>
-
-  <rect x="356" y="494" width="296" height="70" rx="9" class="stat-green"/>
-  <text x="372" y="514" class="stat-cap">Talker GPU</text>
-  <text x="372" y="538" class="stat-big">next step starts immediately</text>
-  <text x="372" y="555" class="stat-sub">decode workers stay busy</text>
-
-  <rect x="668" y="494" width="312" height="70" rx="9" class="stat-gray"/>
-  <text x="684" y="514" class="stat-cap">Background builder</text>
-  <text x="684" y="538" class="stat-big">materializes Omni output</text>
-  <text x="684" y="555" class="stat-sub">payload work overlaps the next Talker step</text>
-</svg>
@@ -15,7 +15,7 @@ The full set of backends and their platform defaults is in the **Backend Options
 | Value | Notes |
 |---|---|
 | `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Supports **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
-| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. |
+| `FLASH_ATTN` | Wraps FlashAttention 2. Default on Hopper / Ada / Ampere when `flash-attn` is installed. |
 | `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
 | `FLASHINFER_ATTN` | Calls FlashInfer's dense `single_prefill_with_kv_cache` directly with `custom_mask` for non-causal masked attention. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer`. |
 | `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
@@ -167,48 +167,6 @@ sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `T
 Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting
 `TRTLLM_ATTN` raises rather than silently degrading.

-## TRTLLM_ATTN SAGE Quantization
-
-On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen
-FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run
-in low precision. Enable it through the typed `quant` block on the attention spec:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). |
-| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). |
-| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). |
-
-V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so
-neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels.
-
-The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the
-fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),
-`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),
-`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'
-```
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16),
-    ),
-)
-```
-
-
-**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel
-availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**
-(B300); `int8` QK kernels are compiled for **SM100 only**.
-
 ## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)

 Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
@@ -235,17 +193,9 @@ DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/te
    --model Lightricks/LTX-2 ...
 ```

-### FlashAttention-4 on Blackwell
+### FA4 not yet integrated

-Install the optional CUDA 13 extra to use the CuTe-based FA4 path:
-
-```bash
-pip install 'vllm-omni[fa4]'
-```
-
-`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2
-when it is unavailable. Version `4.0.0b18` is required; earlier beta
-wheels had known JIT failures on Blackwell.
+FlashAttention-4 (released March 2026) targets Blackwell natively and reportedly beats cuDNN by ~20% on B200. As of this writing the `flash-attn-4 4.0.0b10` wheel crashes with `AttributeError: 'NoneType' object has no attribute '_trait'` during JIT on sm_120. Not yet wired into vLLM-Omni; revisit when stable lands.

 ## Choosing a Backend Manually

@@ -403,3 +353,64 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_t
    --tensor-parallel-size 2 \
    --output outputs/hv15_sage3.mp4
 ```
+
+### Mixed backends across roles
+
+Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --diffusion-attention-config.per_role.self.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_mixed.mp4
+```
+
+### Compare against FlashAttention
+
+Unset the backend override, or explicitly use `FLASH_ATTN`:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --height 704 --width 1280 --num-frames 49 \
+    --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_fa3.mp4
+```
+
+## Validation Guidance
+
+Don't assume a faster attention backend is numerically interchangeable with `TORCH_SDPA`.
+
+Always compare:
+
+- End-to-end runtime
+- Diffusion-stage runtime (`add_req_and_wait` line in DiffusionEngine.step breakdown)
+- Output quality against a known-good baseline (CLIP similarity, frame-level diff, or visual review)
+
+At minimum, keep the same:
+
+- model
+- prompt
+- seed
+- resolution
+- frame count / step count
+- parallel config (TP / CFG-parallel / Ulysses degrees)
+
+## Reproducing the Benchmark Table
+
+The end-to-end numbers above were collected by running `text_to_video.py` /
+`text_to_image.py` with the same prompt and seed while varying
+`DIFFUSION_ATTENTION_BACKEND`. For a quick kernel-level comparison of the
+backends without loading a model:
+
+```bash
+python benchmarks/diffusion/bench_attention_backends.py --preset hv15
+```
+
+It runs all three BF16 backends on representative DiT attention shapes and
+prints a ranking table at the end.
@@ -179,134 +179,7 @@ class Flux2Transformer2DModel(nn.Module):
 - Support single GPU only for now


-## Distributed Layerwise Offloading
-
-### How It Works
-
-Distributed layerwise offloading extends single-GPU layerwise offloading to
-multi-device deployments.  Each DP rank stores only **1/dp_size** of the model
-weights in host memory; full layer weights are reconstructed at runtime via
-**AllGather** on a dedicated communication stream, overlapped with computation
-via a fixed double-buffer scheme.
-
-**Key features:**
- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights,
-  reconstructed per-layer via `all_gather_into_tensor`
- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any
-  time, regardless of model size
- **DP multi-concurrency**: N concurrent requests processed in parallel
-  (AllGather only gathers weight shards, which are request-independent)
- **mmap weight loading**: weights loaded as mmap views pointing to shared OS
-  page cache, eliminating O(dp_size × model_size) RSS during model creation
- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU
-  (CANN/HCCL) via vLLM-Omni's platform abstraction
-
-**Execution Flow (per device):**
-
-```
-Compute Stream:  [Layer N]          [Layer N+1]          [Layer N+2]
-H2D Stream:      [H2D Shard N+1]   [H2D Shard N+2]
-AllGather:       [AG N+1]          [AG N+2]
-Slot Usage:      Slot0: Layer N    Slot1: Layer N+1
-```
-
-### Usage
-
-**CLI:**
-```bash
-# 4× GPU/NPU with AllGather (recommended)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4
-
-# Without AllGather (each rank loads full weights, no sharding)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4 \
-  --dlo-no-use-allgather
-
-# With SP instead of DP (long sequences)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --usp 4
-```
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(
-    model="/path/to/model",
-    enable_distributed_layerwise_offload=True,
-    dlo_use_allgather=True,  # default
-)
-```
-
-### CLI Flags
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
-| `--data-parallel-size N` | Number of DP ranks for weight sharding | `1` |
-| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` |
-| `--dlo-no-use-allgather` | Full weights per rank (no sharding, no AllGather) | `false` |
-
-### How model weights are loaded (mmap path)
-
-When DLO + AllGather is active, the offloader:
-
-1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`)
-   from the normally-created transformer
-2. **Converts to meta device** via `to_empty(device="meta")`, releasing random
-   initialization weights
-3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`,
-   which return views into the OS page cache (shared across all ranks, 0 RSS)
-4. **Calls `post_load_weights()`** to apply model-specific dtype conversions
-   (e.g. Cosmos3's `time_embedder` → FP32)
-5. **Restores non-persistent buffers** from saved copies
-
-This approach requires **zero model-specific code changes** — no pipeline or
-transformer modifications are needed.
-
-### OffloadPlan (declarative topology metadata)
-
-Models can optionally declare an `OffloadPlan` class variable to provide
-topology metadata (block attribute names, submodules to offload) without
-any offload-specific logic:
-
-```python
-from vllm_omni.diffusion.offloader import OffloadPlan
-
-class MyPipeline(nn.Module):
-    _dit_modules = ["transformer"]
-    _offload_plan = OffloadPlan(
-        block_attrs={"transformer": ("blocks",)},
-        offload_submodules={"context_encoder": "layers"},
-    )
-```
-
-When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs`
-and heuristic attribute search.  This is backward-compatible — existing models
-work without any changes.
-
-### DP Multi-concurrency
-
-When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches
-up to `dp_size` requests per denoise step.  Each DP rank processes a different
-request while AllGather synchronizes only weight shards (request-independent).
-
-**Requirements for concurrent requests:**
- `num_inference_steps` must be specified explicitly (None is not allowed)
- All concurrent requests must have the same `num_inference_steps` value
- These constraints exist because AllGather is a collective that requires
-  every rank to participate at each denoise step
-
-### Limitations
- Online quantization (FP8) is incompatible with mmap loading — falls back to
-  regular `load_weights()` automatically
- Tensor Parallel is not supported (DLO uses DP-based sharding)
- HSDP + AllGather is rejected (would double-shard weights)
- `num_inference_steps=None` is not allowed in DP multi-concurrency mode
+### Implementation Notes

 **Module Discovery**

@@ -335,10 +208,7 @@ OffloadBackend (base class)
 ├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap)
 │                              (delegates to a pipeline's enable_omni_model_cpu_offload
 │                               for split models like Cosmos3)
-├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
-│                          (single-GPU, full weights on host)
-└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook
-                                         (multi-GPU, 1/dp_size sharded weights + AllGather)
+└── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
 ```

 Factory function `get_offload_backend()` selects the appropriate backend based on
@@ -351,22 +221,20 @@ reasoner/generator components inside the model forward pass.

 ## Supported Models

-| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) |
-|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------|
-| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` |
-| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` |
-| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` |
-| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` |
-| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` |
-| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` |
-| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` |
+| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Blocks Attrs (Layerwise specific) |
+|--------------|----------------|-----------|---------------------|-------------------|-----------------------------------|
+| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | `"transformer_blocks"`, `"single_transformer_blocks"` |
+| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | `"layers"` |
+| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | `"transformer"` |
+| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | `"transformer_blocks"` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | `"down_blocks"`, `"up_blocks"` |
+| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
+| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
+| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
+| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |

 **Notes:**
 - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
 - Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
- Distributed Layerwise Offloading works with any model that supports Layerwise Offloading — no additional model changes required.  See [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md) for usage examples.
@@ -1,252 +0,0 @@
-# Diffusion Execution Modes
-
-vLLM-Omni supports complete-request execution and step-wise diffusion
-execution. Both modes use the same asynchronous engine output stream; the
-configuration controls how work is scheduled and whether intermediate outputs
-are exposed.
-
-## Choose a Mode
-
-| Goal | CLI configuration |
-|---|---|
-| Serial request execution | `--max-num-seqs 1` |
-| Fused request-level batching | `--max-num-seqs N` |
-| Single-request step execution | `--step-execution --max-num-seqs 1` |
-| Step-wise continuous batching | `--step-execution --max-num-seqs N` |
-| Chunked diffusion output | `--diffusion-streaming-output` |
-
-`N` must be greater than `1` to allow batching. The selected pipeline must
-support request-level batching or batched-step execution as appropriate;
-single-request step support alone is not sufficient for step-wise continuous
-batching.
-
-## Request Execution
-
-Request execution is the default when `--step-execution` is omitted. The
-pipeline performs a complete `forward()` for each scheduler wave.
-
-### Serial Requests
-
-Use the conservative serial path with any request-mode pipeline:
-
-```bash
-vllm serve MODEL --omni \
-  --port 8091 \
-  --max-num-seqs 1
-```
-
-### Request-Level Batching
-
-Set `max_num_seqs` above one to combine compatible independent requests into a
-single pipeline forward:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4
-```
-
-For bursty traffic, add a small admission window:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --max-num-seqs 4 \
-  --request-batch-max-wait-ms 20
-```
-
-`--request-batch-max-wait-ms 0` is the default. A nonzero value can improve
-batch formation but adds up to that much latency before a new scheduler wave.
-It has no effect in step mode.
-
-Each prompt remains a separate logical request. Do not submit a top-level list
-as one packed prompt. Concurrent serving requests are batched internally when
-their shapes, guidance settings, output counts, and LoRA settings are
-compatible.
-
-Only pipelines that declare request-batch support accept
-`max_num_seqs>1` in request mode. Unsupported pipelines fail during engine
-initialization; use `max_num_seqs=1` instead.
-
-## Step Execution
-
-Step execution lets the scheduler advance and abort requests between denoise
-steps:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 1
-```
-
-Set a larger capacity to allow compatible requests to share denoise waves:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
-```
-
-Step continuous batching is experimental. Start with `max_num_seqs=1` when
-validating a model or debugging correctness, then increase it for
-multi-request throughput.
-
-Step execution is capability-based, not a generic switch for every diffusion
-model. Qwen-Image supports step-wise continuous batching. HunyuanImage3 also
-supports it, but only when its resolved self-attention backend is `TORCH_SDPA`;
-set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` or configure
-`diffusion_attention_config.default.backend=TORCH_SDPA` before using
-`--max-num-seqs >1`. See the
-[HunyuanImage-3.0 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Tencent/HunyuanImage-3.0-Instruct.md)
-for its validated configuration. Helios supports single-request step execution only: use
-`--step-execution --max-num-seqs 1` for Helios. Consult the selected pipeline's
-documentation and source for the latest support status.
-
-## Streaming Output
-
-Use `--diffusion-streaming-output` for a pipeline that can produce intermediate
-diffusion outputs:
-
-```bash
-vllm serve BestWishYsh/Helios-Distilled --omni \
-  --port 8000 \
-  --diffusion-streaming-output
-```
-
-Streaming output requires step execution. If
-`--diffusion-streaming-output` is set without `--step-execution`, the engine
-enables step execution automatically. Model initialization fails if the
-pipeline does not implement step execution.
-
-Chunk-capable pipelines emit intermediate and final outputs through the same
-request stream. Final-only step pipelines emit only the final result even when
-streaming output is enabled.
-
-The normal non-streaming serving path uses the same internal stream but drains
-it and returns only the final output. Users do not need to enable
-`--diffusion-streaming-output` to benefit from the unified engine lifecycle.
-
-## Send Requests
-
-Execution-mode flags configure the server. They do not change the
-OpenAI-compatible client request format. For example, after starting any of
-the non-streaming configurations above:
-
-```bash
-curl -X POST http://localhost:8091/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "prompt": "a cat sitting on a windowsill",
-    "size": "1024x1024",
-    "num_inference_steps": 50,
-    "seed": 42
-  }' | jq -r '.data[0].b64_json' | base64 -d > cat.png
-```
-
-To create a request-level batch, send multiple independent requests
-concurrently. Do not put several prompts into one `prompt` field:
-
-```bash
-for prompt in "a red fox" "a blue bird" "a green frog"; do
-  curl -s -X POST http://localhost:8091/v1/images/generations \
-    -H "Content-Type: application/json" \
-    -d "{\"prompt\":\"${prompt}\",\"size\":\"1024x1024\",\"seed\":42}" \
-    > /dev/null &
-done
-wait
-```
-
-The scheduler may batch these requests when their sampling parameters are
-compatible. See the
-[Image Generation API](../../serving/image_generation_api.md) for response
-formats and additional client examples. Streaming models can use
-model-specific streaming endpoints documented by their serving guide.
-
-## Python API
-
-The Python arguments mirror the CLI flags:
-
-```python
-from vllm_omni import Omni
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    max_num_seqs=4,
-    request_batch_max_wait_ms=20.0,
-)
-
-outputs = omni.generate(
-    [
-        "a cup of coffee on a table",
-        "a toy dinosaur on a sandy beach",
-        "a fox waking up in bed and yawning",
-    ]
-)
-```
-
-Each list item passed to `Omni.generate()` becomes an independent logical
-request that the scheduler may batch with compatible items.
-
-For step execution:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    step_execution=True,
-    max_num_seqs=1,
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-## Stage Configuration
-
-Use the equivalent engine arguments in a deployment YAML:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    stage_type: diffusion
-    engine_args:
-      step_execution: false
-      max_num_seqs: 4
-      request_batch_max_wait_ms: 20
-```
-
-For step execution, set `step_execution: true` and remove
-`request_batch_max_wait_ms`.
-
-## CLI Reference
-
-| Flag | Default | Effect |
-|---|---:|---|
-| `--step-execution` | disabled | Select step-wise scheduling |
-| `--max-num-seqs` | `1` for diffusion stages | Set request- or step-scheduler capacity |
-| `--request-batch-max-wait-ms` | `0` | Wait for burst coalescing in request mode |
-| `--diffusion-streaming-output` | disabled | Expose supported intermediate diffusion outputs and require step execution |
-
-## Limitations and Troubleshooting
-
- Requests batch only when their compatibility-sensitive parameters match.
- Different LoRA adapters or scales run in separate batches.
- FIFO scheduling can cause an incompatible request to block later compatible
-  requests.
- All diffusion cache backends are unsupported in step mode. KV transfer is
-  supported for newly admitted step requests; some other request-mode extras
-  remain unsupported.
- If request-mode startup reports that the pipeline does not support batching,
-  use `--max-num-seqs 1`.
- If step-mode startup mentions `prepare_encode()`, `denoise_step()`,
-  `step_scheduler()`, or `post_decode()`, the pipeline does not implement the
-  required step contract.
-
-For implementation details and model-author guidance, see
-[Diffusion Continuous Batching](../../design/feature/diffusion_continuous_batching.md).
@@ -0,0 +1,119 @@
+# Request-Level Batching
+
+Request-level batching lets diffusion serving combine multiple compatible
+logical requests into one pipeline `forward()` call. Each prompt remains a
+separate request with its own `request_id`, sampling parameters, seed, output,
+error, and abort state. The scheduler decides which requests can run together.
+
+!!! warning "Prompt List Semantics"
+
+    Diffusion request-level batching does not support a top-level packed
+    list-prompt request. Submit multiple prompts as independent requests and let
+    the scheduler batch compatible in-flight requests. Multimodal payloads stay
+    inside a single prompt dict, for example
+    `{"prompt": "...", "multi_modal_data": {"image": image}}`.
+
+## Enablement
+
+Request-level execution is the default diffusion request mode when
+`step_execution=False`. `max_num_seqs` controls the maximum number of compatible
+requests in one scheduler wave:
+
+- `max_num_seqs=1` is the serial request path.
+- `max_num_seqs>1` allows fused request-level batching for pipelines that
+  declare request-batch support.
+
+Increase `max_num_seqs` above `1` to allow the request scheduler to keep more
+than one compatible request active:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4
+```
+
+For bursty online traffic, you can also set a small admission wait window. This
+lets the engine wait briefly before the first `schedule()` of a new wave so
+nearby compatible requests can arrive and share the same fused forward pass:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+`--request-batch-max-wait-ms 0` is the default and disables admission waiting,
+so there is no added wait latency.
+
+For deploy YAMLs, configure the diffusion stage engine args:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    stage_type: diffusion
+    engine_args:
+      max_num_seqs: 4
+      request_batch_max_wait_ms: 20
+```
+
+## Compatibility
+
+Only pipelines that declare request-batch support can use the fused
+multi-request `pipeline.forward(batch)` path. Pipelines that do not support this
+contract can still use request-level execution with `max_num_seqs=1`, which is
+the serial request path. Setting `max_num_seqs>1` for a pipeline without
+request-batch support fails early during engine initialization.
+
+The scheduler batches only compatible requests. Compatibility is based on
+shape-sensitive and guidance-sensitive sampling fields, including resolution,
+frame count, CFG settings, output count, and LoRA identity. Requests with
+different LoRA adapters or scales are kept in separate batches so the worker
+activates one adapter per batch.
+
+Request-level batching applies when `step_execution=False`. For the separate
+step-wise runtime, see [Step Execution](step_execution.md).
+
+## Tuning
+
+- `max_num_seqs` caps the number of active compatible requests in one scheduler
+  wave. `1` means serial request execution; values above `1` enable fused
+  request-level batching when the pipeline supports it.
+- `request_batch_max_wait_ms` is an upper bound on extra admission wait before a
+  new wave starts. Keep it small for latency-sensitive serving; values such as
+  `10` to `50` ms are a practical starting range for bursty HTTP ingress.
+- `0` disables admission waiting and preserves the lowest first-request latency.
+- FIFO ordering is conservative: an incompatible request at the front of the
+  waiting queue can block later compatible requests from joining the current
+  batch.
+
+## Python API
+
+When constructing `Omni`, pass the same engine arguments:
+
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    max_num_seqs=4,
+    request_batch_max_wait_ms=20.0,
+)
+
+outputs = omni.generate(
+    [
+        "a cup of coffee on a table",
+        "a toy dinosaur on a sandy beach",
+        "a fox waking up in bed and yawning",
+    ]
+)
+```
+
+`Omni.generate([...])` submits each list item as its own logical diffusion
+request. The runtime may batch those requests internally when their sampling
+parameters are compatible.
+
+## For Contributors
+
+For implementation details and model-author guidance, see
+[Request-Level Batching for Diffusion](../../design/feature/diffusion_request_level_batching.md).
@@ -0,0 +1,100 @@
+# Step Execution
+
+Step execution is an opt-in diffusion execution mode enabled with
+`step_execution=True` when constructing `Omni`.
+
+It is not a generic diffusion toggle for every pipeline. Only pipelines that
+implement the stepwise contract support it today.
+
+## Quick Start
+
+### Python API
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    step_execution=True,
+)
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(
+        num_inference_steps=50,
+    ),
+)
+```
+
+### Serving
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+For serving, `--step-execution` enables the step-wise runtime. Continuous
+batching only becomes relevant when `--max-num-seqs > 1`.
+
+`max_num_seqs` controls the step scheduler capacity:
+
+- `max_num_seqs=1` runs one request through the step-wise state machine.
+- `max_num_seqs>1` allows compatible active requests to share step waves.
+
+## Supported Pipelines
+
+| Pipeline | Example models | Step execution |
+|----------|----------------|----------------|
+| `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
+| All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
+
+!!! warning "Experimental continuous batching"
+    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
+    the step-wise path can batch
+    compatible requests together. This is experimental. Requests with
+    incompatible sampling parameters are intentionally kept in separate batches,
+    and `max_num_seqs=1` remains the conservative single-request step path.
+
+## Current Limitations
+
+- Continuous batching under `step_execution` is experimental and only batches
+  compatible requests.
+- `cache_backend` is not supported together with step execution.
+- Unsupported pipelines fail early during model loading.
+- Request-mode extras such as KV transfer are not wired into step mode yet.
+- LoRA is supported in step mode, but each batch must use a single adapter:
+  requests with different `lora_request` or `lora_scale` are scheduled into
+  separate batches.
+
+## When To Use It
+
+Use step execution only when you specifically need the pipeline to run through
+its stepwise request state machine. For normal diffusion inference, leave it
+disabled unless your workflow depends on this mode.
+
+For Qwen-Image online serving, the usual progression is:
+
+- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
+- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
+
+If you are looking for general diffusion speedups, see
+[Diffusion Features Overview](../diffusion_features.md).
+
+## Troubleshooting
+
+If model loading fails with a message mentioning `prepare_encode()`,
+`denoise_step()`, `step_scheduler()`, and `post_decode()`, the selected
+pipeline does not support step execution.
+
+## For Model Authors
+
+If you want to add step execution support to a new diffusion pipeline, see the
+implementation guide:
+[Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
+
+If you also want that pipeline to participate in the experimental batched
+step-wise path, see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
@@ -80,11 +80,12 @@ denoise steps.

 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[Diffusion Execution Modes](diffusion/execution_modes.md)** | Configures serial requests, request batching, step execution, continuous batching, and streaming output | Matching latency, throughput, cancellation, and output-delivery requirements |
+| **[Request-Level Batching](diffusion/request_batching.md)** | Scheduler batches compatible independent diffusion requests into one pipeline forward pass | Bursty online serving and multi-request throughput |
+| **[Step Execution](diffusion/step_execution.md)** | Per-step denoise execution with mid-request abort support | Request cancellation between denoise steps, fine-grained execution control |

-**Note:** Request-level batching and step execution are capability-based.
-Consult the execution guide and the selected pipeline's documentation for
-current support.
+**Note:** Request-level batching is available for pipelines that declare the
+request-batch forward contract. Step execution is currently supported by
+QwenImagePipeline only. See [Supported Models](#supported-models) for details.

 ### Quantization Methods

@@ -159,7 +160,7 @@ The following tables show which models support each feature:
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
-| **MiniMax-H3**               |     ❌     |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
+

 **Frame Interpolation Support**

@@ -205,7 +206,7 @@ The following tables show which models support each feature:
    3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
    4. CPU Offloading (Layerwise) supports single-card for now.
    5. Using FP8-Quant as an example of qunatization methods.
-    6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
+    6. Step Execution is not compatible with cache backends (TeaCache, Cache-DiT). LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).


 ## Multi-Thread Weight Loading
@@ -285,7 +286,7 @@ Measured on NVIDIA H800:

 **Execution Modes:**

- **[Diffusion Execution Modes](diffusion/execution_modes.md)** - Configure request batching, step execution, continuous batching, and streaming output
+- **[Step Execution Guide](diffusion/step_execution.md)** - Per-step denoise execution with mid-request abort support

 **Startup Optimization:**

@@ -184,7 +184,7 @@ if __name__ == "__main__":
 !!! info

    For diffusion request-level batching controls such as `max_num_seqs`, see
-    [Diffusion Execution Modes](../../diffusion/execution_modes.md).
+    [Request-Level Batching](../../diffusion/request_batching.md).

 ### Negative Prompts

@@ -40,7 +40,7 @@ bash run_server_stage_cli.sh --stage 0
 bash run_server_stage_cli.sh --stage 1
 ```

-To use a custom deploy YAML:
+To use a custom deploy YAML (note: `--stage-configs-path` is deprecated in favor of `--deploy-config`):

 ```bash
 vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 \
@@ -11,8 +11,8 @@ memory, sampling params) live in `vllm_omni/deploy/glm_image.yaml`.
 vllm serve zai-org/GLM-Image --omni --port 8091
 ```

-The config system auto-detects the pipeline from the model's `model_index.json`, so
-no manual `--deploy-config` is needed.
+The config system auto-detects the pipeline from the model's `model_index.json` — no
+manual `--stage-configs-path` or `--deploy-config` needed.

 By default, stage 0 (AR) runs on GPU 0 and stage 1 (Diffusion) on GPU 1. To colocate
 both stages on a single GPU, override per stage:
@@ -15,9 +15,9 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```

-To use a custom deploy config, launch the server with the command below:
+If you have custom stage configs file, launch the server with command below
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```

 ### Send Multi-modal Request
@@ -182,7 +182,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
- `--deploy-config`: Path to a custom deploy config YAML file (optional)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -195,9 +195,9 @@ The script supports the following arguments:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```

-To use a custom deploy config:
+If you have custom stage configs file:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```

 **Step 2: Run the Gradio demo**
@@ -306,7 +306,7 @@ The gradio script supports the following arguments:
    ``````
 ??? abstract "qwen3_omni_moe_thinking.yaml"
    ``````yaml
-    --8<-- "vllm_omni/deploy/qwen3_omni_moe_thinking.yaml"
+    --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
    ``````
 ??? abstract "run_curl_multimodal_generation.sh"
    ``````sh
@@ -422,19 +422,12 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.

 ### Streaming WebSocket
-The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```

-`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
-```bash
-python qwen3_tts/streaming_speech_client.py \
-    --text "First utterance." \
-    --text "Second utterance, same connection."
-```
-
 ### Gradio demos
 ```bash
 ./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)
@@ -69,19 +69,19 @@ The behaviors and conclusions above are summarized from current fault injection

 ### Example: `SIGTERM` fault injection log (Qwen3-Omni)

-The following log is based on a real run where a `SIGTERM` signal was sent to the serve root process. Its command line has been normalized to the current `--deploy-config` option.
+The following full log is from a real run where a `SIGTERM` signal was sent to the serve root process.

 ```text
-Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+Launching OmniServer with: /workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 Server ready on 127.0.0.1:60675 (OmniServer startup took 206.023s)
 OmniServer started successfully
-[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] current_server_proc pid=556855 name=python3 cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [reliability][process-kill] current_server_proc pid=557118 name=python3 cmdline=/workspace/.venv/bin/python3 -c from multiprocessing.resource_tracker import main;main(57)
 [reliability][process-kill] current_server_proc pid=557119 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=557122 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0
 [reliability][process-kill] current_server_proc pid=558201 name=VLLM::StageEngineCoreProc_noid_replica0_DP0 cmdline=VLLM::StageEngineCoreProc_noid_replica0_DP0

-[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --deploy-config vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
+[reliability][process-kill] root-kill pid=556855 name=python3 signal=SIGTERM cmdline=/workspace/.venv/bin/python3 -m vllm_omni.entrypoints.cli.main serve Qwen/Qwen3-Omni-30B-A3B-Instruct --host 127.0.0.1 --port 60675 --omni --async-chunk --stage-init-timeout 600 --init-timeout 900 --log-stats --stage-configs-path vllm-omni/vllm_omni/deploy/qwen3_omni_moe.yaml
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [omni_base.py:463] [AsyncOmni] Shutting down
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [async_omni_engine.py:2397] [AsyncOmniEngine] Shutting down Orchestrator
 [0;36m(APIServer pid=556855)[0;0m INFO 05-19 12:24:51 [orchestrator.py:351] [Orchestrator] Received shutdown signal
@@ -93,7 +93,6 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
 | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
 | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | |
-| MiniMax H3 | `MiniMaxAI/MiniMax-H3` | Yes | Not validated | DiT linears except patch, timestep, and final projections; not compatible with layerwise offload | None | |

 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)

@@ -8,4 +8,5 @@ mimo_audio: MiMo-Audio
 ming_flash_omni: Ming-flash-omni 2.0
 qwen2_5_omni: Qwen2.5-Omni
 qwen3_omni: Qwen3-Omni
+sensenova_u1: SenseNova-U1
 step_audio2: Step-Audio2
@@ -334,7 +334,6 @@ def main():
    is_ltx23 = "ltx23" in model_class_name_lower or "ltx-2.3" in model_name
    is_ltx2 = is_ltx2_distilled or is_ltx23 or "ltx2" in model_class_name_lower or "ltx-2" in model_name
    is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower())
-    is_cosmos_edge = is_cosmos and ("edge" in model_name or "edge" in model_class_name_lower)

    image = PIL.Image.open(args.image).convert("RGB") if args.image else None
    last_image = PIL.Image.open(args.last_image).convert("RGB") if args.last_image else None
@@ -348,19 +347,7 @@ def main():

    # Per-model generation defaults, applied only when the matching flag is omitted.
    # Cosmos3 would otherwise silently inherit the Wan2.2 defaults (wrong size/steps/shift).
-    if is_cosmos_edge:
-        # Cosmos3-Edge native defaults: 480x832, gs 5.0, flow_shift 3.0 — NOT the Nano/Super
-        # 720p / gs 6.0 / flow_shift 10.0 below, which produce degenerate output on Edge.
-        d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
-            24,
-            5.0,
-            189,
-            35,
-            3.0,
-            480 * 832,
-            16,
-        )
-    elif is_cosmos:
+    if is_cosmos:
        d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
            24,
            6.0,
@@ -1,77 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Offline PersonaPlex inference through the vllm-omni fullduplex backend.
-
-This is the runnable end-to-end proof for the integration: it drives a user WAV
-through :class:`PersonaPlexEngine` + :class:`PersonaPlexSession` (our wrappers
-around the Moshi/Mimi lockstep loop) and writes the agent's audio + inner-monologue
-text. It mirrors ``python -m moshi.offline`` but goes through our ``FrameStepper``
-seam, so a green run validates that the session/adapter plumbing is faithful.
-
-Prereqs: HF access to ``nvidia/personaplex-7b-v1`` (``export HF_TOKEN=...``). The
-stack is moshi-free (native temporal + depformer + streaming Mimi via
-``transformers``); decoding is greedy.
-
-Example:
-    python examples/offline_inference/personaplex/personaplex_offline.py \\
-        --input-wav input_assistant.wav --output-wav out.wav \\
-        --voice-prompt NATF2.pt --persona "You are a wise and friendly teacher."
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-
-import numpy as np
-
-from vllm_omni.experimental.fullduplex.personaplex import (
-    PersonaPlexConfig,
-    PersonaPlexEngine,
-    PersonaPlexSession,
-)
-from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--input-wav", required=True, help="User-side input WAV")
-    parser.add_argument("--output-wav", required=True, help="Agent audio output WAV")
-    parser.add_argument("--output-text", default=None, help="Optional JSON for agent text")
-    parser.add_argument("--voice-prompt", default="NATF2.pt", help="Voice clone (bundled .pt or .wav)")
-    parser.add_argument("--persona", default=DEFAULT_PERSONA, help="System role text")
-    parser.add_argument("--hf-repo", default="nvidia/personaplex-7b-v1")
-    parser.add_argument("--device", default="cuda")
-    parser.add_argument("--cpu-offload", action="store_true")
-    args = parser.parse_args()
-
-    import sphn  # local import: only the runnable driver needs the audio I/O dep
-
-    config = PersonaPlexConfig(
-        hf_repo=args.hf_repo,
-        voice_prompt=args.voice_prompt,
-        persona=args.persona,
-        device=args.device,
-        cpu_offload=args.cpu_offload,
-    )
-
-    engine = PersonaPlexEngine(config).load()
-    session = PersonaPlexSession(engine, config)
-    session.open()
-
-    # Load exactly like the reference (moshi.lm.load_audio): native read then
-    # explicit resample, so greedy runs stay bit-faithful to `python -m moshi.offline`.
-    user_pcm, src_sr = sphn.read(args.input_wav)
-    user_pcm = sphn.resample(user_pcm, src_sample_rate=src_sr, dst_sample_rate=config.sample_rate)
-    audio, text = session.run(np.asarray(user_pcm[0], dtype=np.float32))
-
-    sphn.write_wav(args.output_wav, audio, config.sample_rate)
-    print(f"wrote {audio.shape[0] / config.sample_rate:.2f}s of agent audio to {args.output_wav}")
-    print(f"agent text: {text!r}")
-    if args.output_text:
-        with open(args.output_text, "w") as f:
-            json.dump({"text": text}, f, ensure_ascii=False)
-
-
-if __name__ == "__main__":
-    main()
@@ -260,7 +260,7 @@ if __name__ == "__main__":
    For diffusion pipelines, the input list is sliced into single-item requests
    before feeding into the diffusion pipeline. For request-level batching
    controls such as `max_num_seqs`, see
-    [Diffusion Execution Modes](../../../docs/user_guide/diffusion/execution_modes.md).
+    [Request-Level Batching](../../../docs/user_guide/diffusion/request_batching.md).

 ### Negative Prompts

@@ -182,13 +182,10 @@ def run_e2e():
                print(f"Multimodal output keys: {mm.keys()}")
                if "audio" in mm:
                    audio_out = mm["audio"]
-                    # The model emits its own rate (24 kHz for CosyVoice3); take
-                    # it from the payload so the file is not mislabeled.
-                    sample_rate = int(mm["sr"]) if "sr" in mm.keys() else 24000
                    print(f"Generated Audio Shape: {audio_out.shape}")
                    out_path = f"output_{i}.wav"
-                    sf.write(out_path, audio_out.cpu().numpy().squeeze(), sample_rate)
-                    print(f"Saved audio to {out_path} (sr={sample_rate})")
+                    sf.write(out_path, audio_out.cpu().numpy().squeeze(), 22050)
+                    print(f"Saved audio to {out_path}")
            else:
                print("No multimodal output found.")
        except Exception as e:
@@ -56,19 +56,6 @@ _MODEL_PRESETS = {
        "flow_shift": 10.0,
        "output": "cosmos3_t2v_output.mp4",
    },
-    # Cosmos3-Edge is a compact checkpoint with its own native resolution / guidance /
-    # flow_shift. It must NOT inherit the Nano/Super "cosmos" numbers above (720p / gs 6.0 /
-    # flow_shift 10.0), which produce degenerate output on Edge.
-    "cosmos3_edge": {
-        "height": 480,
-        "width": 832,
-        "num_frames": 189,
-        "num_inference_steps": 35,
-        "guidance_scale": 5.0,
-        "fps": 24,
-        "flow_shift": 3.0,
-        "output": "cosmos3_edge_t2v_output.mp4",
-    },
    "helios": {
        "height": 384,
        "width": 640,
@@ -116,10 +103,6 @@ def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
        return _MODEL_PRESETS["ltx2"]
    if "vace" in model_lower:
        return _MODEL_PRESETS["vace"]
-    # Edge must be matched before the generic cosmos branch (its "cosmos" substring would
-    # otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
-    if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
-        return _MODEL_PRESETS["cosmos3_edge"]
    if "cosmos" in model_lower:
        return _MODEL_PRESETS["cosmos"]
    if "hunyuan" in model_lower:
@@ -53,5 +53,5 @@ vllm-omni serve "$MODEL" \
    --host 0.0.0.0 \
    --port "$PORT" \
    --trust-remote-code \
-    --deploy-config "$DEPLOY_YAML" \
+    --stage-configs-path "$DEPLOY_YAML" \
    --omni
@@ -28,11 +28,8 @@ If image.command is set, uses that as a full override.
 - {{ .Values.omniArgs.numGpus | quote }}
 {{- end }}
 {{- if .Values.omniArgs.stageConfigsPath }}
-{{- fail "omniArgs.stageConfigsPath has been removed; use omniArgs.deployConfigPath" }}
-{{- end }}
-{{- if .Values.omniArgs.deployConfigPath }}
- "--deploy-config"
- {{ .Values.omniArgs.deployConfigPath | quote }}
+- "--stage-configs-path"
+- {{ .Values.omniArgs.stageConfigsPath | quote }}
 {{- end }}
 {{- if and .Values.omniArgs.cacheBackend (ne .Values.omniArgs.cacheBackend "none") }}
 - "--cache-backend"
@@ -78,26 +78,6 @@ tests:
          path: spec.template.spec.containers[0].command
          content: "--enable-layerwise-offload"

-  - it: should include deploy config in command
-    set:
-      omniArgs:
-        deployConfigPath: "/configs/deploy.yaml"
-    asserts:
-      - equal:
-          path: spec.template.spec.containers[0].command[8]
-          value: "--deploy-config"
-      - equal:
-          path: spec.template.spec.containers[0].command[9]
-          value: "/configs/deploy.yaml"
-
-  - it: should reject removed stage config value
-    set:
-      omniArgs:
-        stageConfigsPath: "/configs/legacy.yaml"
-    asserts:
-      - failedTemplate:
-          errorMessage: "omniArgs.stageConfigsPath has been removed; use omniArgs.deployConfigPath"
-
  - it: should use full command override when image.command is set
    set:
      image:
@@ -67,8 +67,8 @@ omniArgs:
  enableCpuOffload: false
  # -- Number of GPUs for diffusion inference (empty = auto)
  numGpus:
-  # -- Path to deploy config file (empty = auto-detected from model)
-  deployConfigPath:
+  # -- Path to stage configs file (empty = auto-detected from model)
+  stageConfigsPath:
  # -- Cache backend: none, tea_cache, or cache_dit
  cacheBackend: "none"
  # -- Default sampling params as a JSON string
@@ -23,18 +23,17 @@ including `librosa`.

 The deploy config auto-loads via `--omni`.
 The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
-logical device 0 with memory budgets of 55%, 15%, and 15%, leaving headroom
-for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
-profile admits at most four sequences per stage and bounds startup video
-profiling to 32 frames per video. For throughput,
-`minicpmo_4_5_2gpu.yaml` gives the Thinker
+logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits
+at most four sequences per stage and bounds startup video profiling to 32
+frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker
 GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
 profile admits at most four concurrent sequences per stage.

 | deploy config | GPUs | Notes |
 |---|---|---|
 | `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
-| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
+| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
+| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. |
 | `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
 | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
 | `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
@@ -90,18 +89,17 @@ python openai_chat_completion_client_for_multimodal_generation.py \
    --prompt "Briefly introduce yourself."
 ```

-Streaming text + audio (use `--stream`):
+Streaming text + audio:

 ```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-    --query-type text \
-    --prompt "Briefly introduce yourself." \
-    --port 8099 \
-    --stream
+python streaming_chat_completion.py \
+    --base-url http://localhost:8099/v1 \
+    --output minicpmo_stream.wav
 ```

-The client prints text deltas as they arrive and saves streamed audio chunks
-to WAV files.
+The example prints text deltas immediately and joins the independently encoded
+24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip
+Talker and Code2Wav.

 Shared helpers also work if you pass MiniCPM defaults yourself:

@@ -6,7 +6,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
    this.playedFrames = 0;
    this.underrunFrames = 0;
    this.drain = null;
-    this.terminalFadeFrames = 0;
    this.started = false;
    this.activeResponseId = null;
    this.initialBufferFrames = Math.round(sampleRate * 0.2);
@@ -32,7 +31,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
      }
    } else if (message.type === 'drain') {
      this.drain = { responseId: message.responseId || null };
-      this.terminalFadeFrames = Math.min(this.fadeFrames, this.bufferedFrames());
      if (!this.started && this.bufferedFrames() > 0) {
        this.bufferWaitFrames = 0;
        this.startPlayback();
@@ -44,7 +42,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
      this.playedFrames = 0;
      this.underrunFrames = 0;
      this.drain = null;
-      this.terminalFadeFrames = 0;
      this.started = false;
      this.activeResponseId = null;
      this.bufferWaitFrames = this.initialBufferFrames;
@@ -79,7 +76,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
    this.playedFrames = 0;
    this.underrunFrames = 0;
    this.drain = null;
-    this.terminalFadeFrames = 0;
    this.started = false;
    this.activeResponseId = null;
    this.bufferWaitFrames = this.initialBufferFrames;
@@ -120,7 +116,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
    while (target < output.length && this.queue.length > 0) {
      const pcm = this.queue[0];
      const count = Math.min(output.length - target, pcm.length - this.offset);
-      const remainingBeforeChunk = this.bufferedFrames();
      for (let index = 0; index < count; index += 1) {
        let sample = pcm[this.offset + index] / 32768;
        if (this.fadeInFrames > 0) {
@@ -128,16 +123,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
          sample *= elapsed / this.fadeFrames;
          this.fadeInFrames -= 1;
        }
-        const remainingFrames = remainingBeforeChunk - index;
-        if (
-          this.drain
-          && this.terminalFadeFrames > 0
-          && remainingFrames <= this.terminalFadeFrames
-        ) {
-          sample *= this.terminalFadeFrames === 1
-            ? 0
-            : (remainingFrames - 1) / (this.terminalFadeFrames - 1);
-        }
        output[target + index] = sample;
      }
      target += count;
@@ -3,7 +3,7 @@
 #
 # Prereq:
 #   Start a vllm-omni OpenAI server for MiniCPM-o 4.5 on :8099 (see the
-#   8x4090 deploy config at vllm_omni/deploy/minicpmo_4_5_8x4090.yaml).
+#   8x4090 stage config under vllm_omni/model_executor/stage_configs).
 set -e

 HERE="$(cd "$(dirname "$0")" && pwd)"
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import wave
+from pathlib import Path
+
+from openai import OpenAI
+
+
+def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]:
+    """Decode one base64 WAV delta and append its PCM frames."""
+    with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav:
+        channels = wav.getnchannels()
+        sample_width = wav.getsampwidth()
+        sample_rate = wav.getframerate()
+        pcm_parts.append(wav.readframes(wav.getnframes()))
+    return channels, sample_width, sample_rate
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", default="http://localhost:8099/v1")
+    parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
+    parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.")
+    parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav"))
+    parser.add_argument("--text-only", action="store_true")
+    args = parser.parse_args()
+
+    modalities = ["text"] if args.text_only else ["text", "audio"]
+    stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create(
+        model=args.model,
+        messages=[{"role": "user", "content": args.prompt}],
+        modalities=modalities,
+        stream=True,
+        extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}},
+    )
+
+    pcm_parts: list[bytes] = []
+    audio_format: tuple[int, int, int] | None = None
+    for chunk in stream:
+        for choice in chunk.choices:
+            content = getattr(choice.delta, "content", None)
+            if not content:
+                continue
+            if getattr(chunk, "modality", None) == "audio":
+                current_format = append_wav_chunk(content, pcm_parts)
+                if audio_format is not None and current_format != audio_format:
+                    raise ValueError(f"Audio format changed: {audio_format} -> {current_format}")
+                audio_format = current_format
+            elif getattr(chunk, "modality", None) == "text":
+                print(content, end="", flush=True)
+
+    print()
+    if pcm_parts and audio_format is not None:
+        channels, sample_width, sample_rate = audio_format
+        with wave.open(str(args.output), "wb") as wav:
+            wav.setnchannels(channels)
+            wav.setsampwidth(sample_width)
+            wav.setframerate(sample_rate)
+            wav.writeframes(b"".join(pcm_parts))
+        print(f"Audio saved to {args.output} ({sample_rate} Hz)")
+
+
+if __name__ == "__main__":
+    main()
@@ -1,84 +0,0 @@
-# PersonaPlex full-duplex online serving
-
-Serve [`nvidia/personaplex-7b-v1`](https://huggingface.co/nvidia/personaplex-7b-v1)
-(a Moshi-based full-duplex speech-to-speech model) with the native vLLM-Omni engine.
-
-The server hosts the **official PersonaPlex web client** at `/` (auto-downloaded from
-the model repo) and implements its WebSocket protocol at `/api/chat`, so you talk to the
-model live in the browser — mic in, agent speech + inner-monologue text out.
-
-> Experimental. Requires a GPU and Hugging Face access to the gated repo
-> (`HF_TOKEN` with access to `nvidia/personaplex-7b-v1`).
-
-## Start the server
-
-```bash
-HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 bash run_server.sh
-# or directly:
-HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \
-    vllm_omni.experimental.fullduplex.personaplex.serving.server --port 8124
-```
-
-## Talk to it
-
- **Browser (recommended):** open `http://localhost:8124/` and allow the microphone.
-  Use headphones so the agent does not hear itself.
- **Headless:** stream a 24 kHz mono WAV and save the reply:
-
-  ```bash
-  python duplex_client.py --url ws://localhost:8124/api/chat --input user.wav --out reply.wav
-  ```
-
-## Unified Realtime API
-
-The default `vllm_omni/deploy/personaplex.yaml` also enables main's engine-owned
-full-duplex control plane:
-
-```bash
-CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
-  /path/to/personaplex-7b-v1 \
-  --omni \
-  --stage-configs-path vllm_omni/deploy/personaplex.yaml
-```
-
-Validate the actual `/v1/realtime?duplex=1` scheduler path with paced 24 kHz
-PCM, two concurrent sessions, overflow admission, per-session slot recycling,
-and non-silent output:
-
-```bash
-python tests/e2e/online_serving/personaplex_realtime_duplex.py \
-  --model /path/to/personaplex-7b-v1 \
-  --input-wav /path/to/speech.wav \
-  --output-dir /tmp/personaplex-realtime-duplex
-```
-
-This is distinct from the standalone `/api/chat` compatibility server above.
-The unified endpoint advertises `supports_barge_in=false`: overlapping speech
-is native model behavior, but destructive output interruption and model-state
-rewind have not been validated for PersonaPlex.
-
-## Protocol (`/api/chat`)
-
-Binary WebSocket messages, first byte is a tag (identical to `moshi.server`):
-
-| Direction | Tag | Payload |
-| --- | --- | --- |
-| server → client | `\x00` | ready handshake (after the system-prompt prefill) |
-| client → server | `\x01` | Opus-encoded mic audio (24 kHz) |
-| server → client | `\x01` | Opus-encoded agent audio |
-| server → client | `\x02` | UTF-8 inner-monologue text |
-
-Connect with query params `text_prompt` (persona) and `voice_prompt` (voice file).
-A raw-PCM endpoint (`/v1/audio/duplex`, JSON `open` + float32 frames) is also available
-for clients that do not want an Opus dependency.
-
-## Notes
-
- **Run the client near the server.** Real-time 80 ms audio is sensitive to network
-  latency/jitter; over a high-latency remote link playback can stutter regardless of
-  engine speed. On localhost it is smooth.
- Greedy decoding by default. One conversation per server by default (a second
-  concurrent connection is rejected with WS close 1013 until the first ends);
-  pass `--batch-size N` to host N concurrent conversations on one engine
-  (elastic batching; connections beyond N are rejected until a slot frees).
- Full runbook: [`recipes/NVIDIA/PersonaPlex.md`](../../../recipes/NVIDIA/PersonaPlex.md).
@@ -1,114 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Headless client for the PersonaPlex duplex server (official Moshi protocol).
-
-Streams a 24 kHz mono WAV to ``/api/chat`` exactly as the browser client does
-(Opus over WebSocket, tags ``\\x00`` ready / ``\\x01`` audio / ``\\x02`` text),
-collects the agent's reply audio + inner-monologue text, and writes a WAV.
-
-The browser UI at ``http://<host>:<port>/`` is the primary demo; this script is
-for headless testing / scripting.
-
-    python duplex_client.py --url ws://localhost:8124/api/chat \
-        --input user.wav --out reply.wav
-
-Deps: ``websockets``, ``sphn``, ``soundfile``, ``numpy``.
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-
-import numpy as np
-import soundfile as sf
-import sphn
-
-SR = 24000  # PersonaPlex / Mimi codec rate
-
-
-async def converse(url: str, pcm: np.ndarray) -> tuple[np.ndarray, str]:
-    import websockets
-
-    reader = sphn.OpusStreamReader(SR)  # decode agent audio
-    writer = sphn.OpusStreamWriter(SR)  # encode our mic audio
-    text: list[str] = []
-    ready = asyncio.Event()
-    last = {"t": 0.0}
-
-    async with websockets.connect(url, max_size=None) as ws:
-
-        async def receive() -> None:
-            loop = asyncio.get_event_loop()
-            while True:
-                try:
-                    msg = await ws.recv()
-                except Exception:
-                    return
-                if not isinstance(msg, (bytes, bytearray)) or not msg:
-                    continue
-                tag, payload = msg[0], msg[1:]
-                if tag == 0:
-                    ready.set()
-                elif tag == 1:
-                    reader.append_bytes(bytes(payload))
-                    last["t"] = loop.time()
-                elif tag == 2:
-                    text.append(payload.decode("utf8", errors="replace"))
-
-        recv_task = asyncio.create_task(receive())
-        await asyncio.wait_for(ready.wait(), timeout=120)  # waits for system-prompt prefill
-
-        frame = 1920  # 80 ms
-        for i in range(0, len(pcm), frame):
-            chunk = pcm[i : i + frame]
-            if len(chunk) < frame:
-                chunk = np.concatenate([chunk, np.zeros(frame - len(chunk), dtype=np.float32)])
-            writer.append_pcm(chunk)
-            data = writer.read_bytes()
-            if data:
-                await ws.send(b"\x01" + data)
-            await asyncio.sleep(frame / SR)  # pace at real time
-
-        loop = asyncio.get_event_loop()
-        last["t"] = loop.time()
-        while loop.time() - last["t"] < 2.5:  # drain until the agent stops talking
-            await asyncio.sleep(0.1)
-        recv_task.cancel()
-
-    chunks: list[np.ndarray] = []
-    while True:
-        out = reader.read_pcm()
-        if out is None or out.shape[-1] == 0:
-            break
-        chunks.append(np.asarray(out, dtype=np.float32))
-    audio = np.concatenate(chunks) if chunks else np.zeros(0, dtype=np.float32)
-    return audio, "".join(text)
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--url", default="ws://localhost:8124/api/chat")
-    ap.add_argument("--input", required=True, help="24 kHz mono WAV of the user turn")
-    ap.add_argument("--out", default="personaplex_reply.wav")
-    ap.add_argument("--tail", type=float, default=10.0, help="trailing silence (s) for the agent reply")
-    ap.add_argument("--persona", default="")
-    ap.add_argument("--voice", default="NATF2.pt")
-    args = ap.parse_args()
-
-    wav, sr = sf.read(args.input, dtype="float32")
-    if wav.ndim > 1:
-        wav = wav.mean(axis=1)
-    if sr != SR:
-        raise SystemExit(f"input must be {SR} Hz mono; got {sr} Hz. Resample it first.")
-    wav = np.concatenate([wav, np.zeros(int(args.tail * SR), dtype=np.float32)])
-
-    url = f"{args.url}?text_prompt={args.persona}&voice_prompt={args.voice}"
-    audio, text = asyncio.run(converse(url, wav))
-    sf.write(args.out, audio if audio.size else np.zeros(SR, dtype=np.float32), SR)
-    print(f"agent reply: {audio.size / SR:.2f}s -> {args.out}")
-    print(f"inner-monologue: {text.strip()!r}")
-
-
-if __name__ == "__main__":
-    main()
@@ -1,15 +0,0 @@
-#!/bin/bash
-# PersonaPlex full-duplex online serving.
-#
-# Serves the OFFICIAL PersonaPlex web client at / (auto-downloaded from
-# nvidia/personaplex-7b-v1) and the Moshi WebSocket protocol at /api/chat,
-# backed by the native vLLM-Omni engine. Requires HF access to the gated repo.
-
-HOST="${HOST:-0.0.0.0}"
-PORT="${PORT:-8124}"
-
-echo "Starting PersonaPlex duplex server on ${HOST}:${PORT}..."
-echo "Open http://localhost:${PORT}/ for the official web client (run it near the server)."
-
-python -m vllm_omni.experimental.fullduplex.personaplex.serving.server \
-    --host "$HOST" --port "$PORT"
@@ -12,9 +12,9 @@ Please refer to [README.md](../../../README.md)
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```

-To use a custom deploy config, launch the server with the command below:
+If you have custom stage configs file, launch the server with command below
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```

 ### Send Multi-modal Request
@@ -173,7 +173,7 @@ The script supports the following arguments:
 - `--model`: Model name/path (default: Qwen/Qwen2.5-Omni-7B)
 - `--server-port`: Port for vLLM server (default: 8091)
 - `--gradio-port`: Port for Gradio demo (default: 7861)
- `--deploy-config`: Path to a custom deploy config YAML file (optional)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
 - `--server-host`: Host for vLLM server (default: 0.0.0.0)
 - `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
 - `--share`: Share Gradio demo publicly (creates a public link)
@@ -186,9 +186,9 @@ The script supports the following arguments:
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```

-To use a custom deploy config:
+If you have custom stage configs file:
 ```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```

 **Step 2: Run the Gradio demo**
@@ -13,7 +13,7 @@ set -e
 MODEL="Qwen/Qwen2.5-Omni-7B"
 SERVER_PORT=8091
 GRADIO_PORT=7861
-DEPLOY_CONFIG_PATH=""
+STAGE_CONFIGS_PATH=""
 SERVER_HOST="0.0.0.0"
 GRADIO_IP="127.0.0.1"
 GRADIO_SHARE=false
@@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do
            GRADIO_PORT="$2"
            shift 2
            ;;
-        --deploy-config)
-            DEPLOY_CONFIG_PATH="$2"
+        --stage-configs-path)
+            STAGE_CONFIGS_PATH="$2"
            shift 2
            ;;
        --server-host)
@@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do
            echo "  --model MODEL                 Model name/path (default: Qwen/Qwen2.5-Omni-7B)"
            echo "  --server-port PORT            Port for vLLM server (default: 8091)"
            echo "  --gradio-port PORT            Port for Gradio demo (default: 7861)"
-            echo "  --deploy-config PATH          Path to custom deploy config YAML file (optional)"
+            echo "  --stage-configs-path PATH     Path to custom stage configs YAML file (optional)"
            echo "  --server-host HOST            Host for vLLM server (default: 0.0.0.0)"
            echo "  --gradio-ip IP                IP for Gradio demo (default: 127.0.0.1)"
            echo "  --share                       Share Gradio demo publicly"
@@ -87,8 +87,8 @@ echo "=========================================="

 # Build vLLM server command
 SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
-if [ -n "$DEPLOY_CONFIG_PATH" ]; then
-    SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG_PATH")
+if [ -n "$STAGE_CONFIGS_PATH" ]; then
+    SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
 fi

 # Function to cleanup on exit
@@ -0,0 +1,36 @@
+# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
+# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
+# thinker component and do not support audio output.
+#
+# Single stage: Thinker (multimodal understanding + text generation)
+
+# The following config has been verified on 2x H100-80G GPUs.
+stage_args:
+  - stage_id: 0
+    runtime:
+      devices: "0,1"
+    engine_args:
+      model_stage: thinker
+      max_num_seqs: 1
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_type: ar
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      gpu_memory_utilization: 0.9
+      enforce_eager: true
+      trust_remote_code: true
+      engine_output_type: text
+      distributed_executor_backend: "mp"
+      enable_prefix_caching: true
+      hf_config_name: thinker_config
+      tensor_parallel_size: 2
+    final_output: true
+    final_output_type: text
+    is_comprehension: true
+    default_sampling_params:
+      temperature: 0.4
+      top_p: 0.9
+      top_k: 1
+      max_tokens: 2048
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.05
@@ -13,7 +13,7 @@ set -e
 MODEL="Qwen/Qwen3-Omni-30B-A3B-Instruct"
 SERVER_PORT=8091
 GRADIO_PORT=7861
-DEPLOY_CONFIG=""
+STAGE_CONFIGS_PATH=""
 SERVER_HOST="0.0.0.0"
 GRADIO_IP="127.0.0.1"
 GRADIO_SHARE=false
@@ -33,8 +33,8 @@ while [[ $# -gt 0 ]]; do
            GRADIO_PORT="$2"
            shift 2
            ;;
-        --deploy-config)
-            DEPLOY_CONFIG="$2"
+        --stage-configs-path)
+            STAGE_CONFIGS_PATH="$2"
            shift 2
            ;;
        --server-host)
@@ -56,7 +56,7 @@ while [[ $# -gt 0 ]]; do
            echo "  --model MODEL                 Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)"
            echo "  --server-port PORT            Port for vLLM server (default: 8091)"
            echo "  --gradio-port PORT            Port for Gradio demo (default: 7861)"
-            echo "  --deploy-config PATH          Path to custom deploy config YAML file (optional)"
+            echo "  --stage-configs-path PATH     Path to custom stage configs YAML file (optional)"
            echo "  --server-host HOST            Host for vLLM server (default: 0.0.0.0)"
            echo "  --gradio-ip IP                IP for Gradio demo (default: 127.0.0.1)"
            echo "  --share                       Share Gradio demo publicly"
@@ -87,8 +87,8 @@ echo "=========================================="

 # Build vLLM server command
 SERVER_CMD=("vllm" "serve" "$MODEL" "--omni" "--port" "$SERVER_PORT" "--host" "$SERVER_HOST")
-if [ -n "$DEPLOY_CONFIG" ]; then
-    SERVER_CMD+=("--deploy-config" "$DEPLOY_CONFIG")
+if [ -n "$STAGE_CONFIGS_PATH" ]; then
+    SERVER_CMD+=("--stage-configs-path" "$STAGE_CONFIGS_PATH")
 fi

 # Function to cleanup on exit
@@ -336,7 +336,7 @@ vllm serve OpenMOSS-Team/MOSS-TTS-Nano --omni --port 8091
 # or:
 ./moss_tts_nano/run_server.sh
 ```
-The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no `--deploy-config`, `--trust-remote-code`, or `--enforce-eager` flags are needed.
+The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no `--stage-configs-path`, `--trust-remote-code`, or `--enforce-eager` flags are needed.

 ### Sending requests
 ```bash
@@ -537,19 +537,12 @@ curl -X POST http://localhost:8091/v1/audio/speech \
 Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.

 ### Streaming WebSocket
-The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
 ```bash
 python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
 python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
 ```

-`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
-```bash
-python qwen3_tts/streaming_speech_client.py \
-    --text "First utterance." \
-    --text "Second utterance, same connection."
-```
-
 To receive word-level timestamps, launch the server with a forced aligner:
 ```bash
 vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
@@ -7,12 +7,6 @@ Usage:
    # Send full text at once
    python streaming_speech_client.py --text "Hello world. How are you? I am fine."

-    # Several utterances over one connection: input.done flushes each of them
-    # and the connection stays open, so only the first pays the handshake
-    python streaming_speech_client.py \
-        --text "First utterance." \
-        --text "Second utterance, same connection."
-
    # Pick a built-in speaker for CustomVoice models
    python streaming_speech_client.py \
        --text "打开电子枪，关闭扫描，切换到低倍模式。" \
@@ -70,27 +64,18 @@ except ImportError:
    raise SystemExit(1)


-def frame_basename(msg: dict) -> str:
-    """Name a file after the utterance and sentence a frame belongs to.
-
-    Every utterance is sentence 0, so the utterance index is what keeps the
-    files of one connection from overwriting each other.
-    """
-    return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
-
-
-def save_audio_file(output_dir: str, basename: str, response_format: str, chunks: list[bytes]) -> str:
+def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str:
    audio_bytes = b"".join(chunks)
    if response_format == "pcm":
-        raw_path = os.path.join(output_dir, f"{basename}.pcm")
+        raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm")
        with open(raw_path, "wb") as f:
            f.write(audio_bytes)

-        wav_path = os.path.join(output_dir, f"{basename}.wav")
+        wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav")
        save_pcm_wav(wav_path, audio_bytes)
        return wav_path

-    filename = os.path.join(output_dir, f"{basename}.{response_format}")
+    filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}")
    with open(filename, "wb") as f:
        f.write(audio_bytes)
    return filename
@@ -110,207 +95,183 @@ def timestamp_words_text(timestamps: list[dict] | None) -> str:
    return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word"))


-async def send_utterance(ws, text: str, simulate_stt: bool, stt_delay: float) -> None:
-    """Send one utterance's text, then input.done to flush it."""
-    if simulate_stt:
-        words = text.split(" ")
-        for i, word in enumerate(words):
-            chunk = word + (" " if i < len(words) - 1 else "")
-            await ws.send(
-                json.dumps(
-                    {
-                        "type": "input.text",
-                        "text": chunk,
-                    }
-                )
-            )
-            print(f"  Sent: {chunk!r}")
-            await asyncio.sleep(stt_delay)
-    else:
-        await ws.send(
-            json.dumps(
-                {
-                    "type": "input.text",
-                    "text": text,
-                }
-            )
-        )
-        print(f"Sent full text: {text!r}")
-
-    await ws.send(json.dumps({"type": "input.done"}))
-    print("Sent input.done")
-
-
-async def receive_utterance(
-    ws,
-    output_dir: str,
-    response_format: str,
-    word_timestamps: bool,
-    save_chunks: bool,
-) -> None:
-    """Consume frames until session.done marks the flushed utterance complete."""
-    current_label = "utterance 0"
-    current_sentence_text = ""
-    current_chunks: list[bytes] = []
-    current_timestamps: list[dict] = []
-    # Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
-    # the sentence is a failure only if no frame ever carried timestamps.
-    alignment_frame_seen = False
-
-    while True:
-        message = await ws.recv()
-
-        if isinstance(message, bytes):
-            current_chunks.append(message)
-            print(f"  Received audio chunk for {current_label}: {len(message)} bytes")
-        else:
-            # JSON frame
-            msg = json.loads(message)
-            msg_type = msg.get("type")
-
-            if msg_type == "audio.start":
-                current_label = f"utterance {msg['utterance_index']}"
-                current_sentence_text = msg.get("sentence_text", "")
-                current_chunks = []
-                current_timestamps = []
-                alignment_frame_seen = False
-                print(f"  [{current_label}] Generating: {current_sentence_text!r}")
-            elif msg_type == "audio.chunk":
-                audio = base64.b64decode(msg["audio_b64"])
-                current_chunks.append(audio)
-                chunk_timestamps = msg.get("timestamps")
-                # Sentence-level alignment sends a trailing timestamp-only
-                # frame with empty audio; don't write an empty chunk file.
-                if save_chunks and audio:
-                    chunk_base = os.path.join(
-                        output_dir,
-                        f"{frame_basename(msg)}_chunk_{msg['chunk_id']:03d}",
-                    )
-                    chunk_pcm = f"{chunk_base}.pcm"
-                    chunk_wav = f"{chunk_base}.wav"
-                    chunk_json = f"{chunk_base}_timestamps.json"
-                    with open(chunk_pcm, "wb") as f:
-                        f.write(audio)
-                    save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
-                    with open(chunk_json, "w", encoding="utf-8") as f:
-                        json.dump(
-                            {
-                                "utterance_index": msg["utterance_index"],
-                                "sentence_index": msg["sentence_index"],
-                                "chunk_id": msg["chunk_id"],
-                                "chunk_start_ms": msg.get("chunk_start_ms"),
-                                "chunk_end_ms": msg.get("chunk_end_ms"),
-                                "sentence_text": current_sentence_text,
-                                "covered_text": timestamp_words_text(chunk_timestamps),
-                                "timestamps": chunk_timestamps,
-                            },
-                            f,
-                            ensure_ascii=False,
-                            indent=2,
-                        )
-                if chunk_timestamps is None:
-                    print(f"  [{current_label}] chunk {msg['chunk_id']}: {len(audio)} bytes, timestamps unavailable")
-                else:
-                    alignment_frame_seen = True
-                    current_timestamps.extend(chunk_timestamps)
-                    print(
-                        f"  [{current_label}] chunk {msg['chunk_id']}: "
-                        f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
-                    )
-                if save_chunks and audio:
-                    print(f"    saved chunk -> {chunk_wav}")
-            elif msg_type == "audio.done":
-                filename = save_audio_file(
-                    output_dir,
-                    frame_basename(msg),
-                    response_format,
-                    current_chunks,
-                )
-                if word_timestamps:
-                    # `null` on failure, else the accumulated list (`[]` for silence).
-                    sentence_timestamps = current_timestamps if alignment_frame_seen else None
-                    ts_filename = os.path.join(
-                        output_dir,
-                        f"{frame_basename(msg)}_timestamps.json",
-                    )
-                    with open(ts_filename, "w", encoding="utf-8") as f:
-                        json.dump(
-                            {
-                                "utterance_index": msg["utterance_index"],
-                                "sentence_index": msg["sentence_index"],
-                                "sentence_text": current_sentence_text,
-                                "covered_text": timestamp_words_text(sentence_timestamps),
-                                "timestamps": sentence_timestamps,
-                            },
-                            f,
-                            ensure_ascii=False,
-                            indent=2,
-                        )
-                print(
-                    f"  [{current_label}] Done"
-                    f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
-                    f" error={msg.get('error', False)}"
-                    f" -> {filename}"
-                )
-                if word_timestamps:
-                    print(f"  [{current_label}] Timestamps -> {ts_filename}")
-                current_chunks = []
-                current_timestamps = []
-                alignment_frame_seen = False
-            elif msg_type == "session.done":
-                print(f"\nUtterance {msg['utterance_index']} complete: {msg['total_sentences']} sentence(s) generated")
-                return
-            elif msg_type == "error":
-                print(f"  ERROR: {msg['message']}")
-            else:
-                print(f"  Unknown message: {msg}")
-
-
 async def stream_tts(
    url: str,
-    texts: list[str],
+    text: str,
    config: dict,
    output_dir: str,
    simulate_stt: bool = False,
    stt_delay: float = 0.1,
    save_chunks: bool = False,
 ) -> None:
-    """Connect to the streaming TTS endpoint and process audio responses.
-
-    Every text is synthesized over the same connection: input.done flushes an
-    utterance without closing, so only the first one pays the handshake.
-    """
+    """Connect to the streaming TTS endpoint and process audio responses."""
    os.makedirs(output_dir, exist_ok=True)
-    response_format = config.get("response_format", "wav")
-    word_timestamps = bool(config.get("word_timestamps", False))

    async with websockets.connect(url) as ws:
-        # 1. Send session config; it stays in effect for every utterance below.
+        # 1. Send session config
        config_msg = {"type": "session.config", **config}
        await ws.send(json.dumps(config_msg))
        print(f"Sent session config: {config}")

-        for text in texts:
-            # 2. Send text (either all at once or word-by-word), then flush.
-            sender_task = asyncio.create_task(send_utterance(ws, text, simulate_stt, stt_delay))
-            try:
-                await receive_utterance(
-                    ws,
-                    output_dir=output_dir,
-                    response_format=response_format,
-                    word_timestamps=word_timestamps,
-                    save_chunks=save_chunks,
+        # 2. Send text (either all at once or word-by-word)
+        async def send_text():
+            if simulate_stt:
+                words = text.split(" ")
+                for i, word in enumerate(words):
+                    chunk = word + (" " if i < len(words) - 1 else "")
+                    await ws.send(
+                        json.dumps(
+                            {
+                                "type": "input.text",
+                                "text": chunk,
+                            }
+                        )
+                    )
+                    print(f"  Sent: {chunk!r}")
+                    await asyncio.sleep(stt_delay)
+            else:
+                await ws.send(
+                    json.dumps(
+                        {
+                            "type": "input.text",
+                            "text": text,
+                        }
+                    )
                )
-            finally:
-                sender_task.cancel()
-                try:
-                    await sender_task
-                except asyncio.CancelledError:
-                    pass  # Task cancellation is expected during shutdown
+                print(f"Sent full text: {text!r}")

-        # 3. Release the connection once there is nothing left to say.
-        await ws.send(json.dumps({"type": "session.close"}))
-        print("Sent session.close")
+            # 3. Signal end of input
+            await ws.send(json.dumps({"type": "input.done"}))
+            print("Sent input.done")
+
+        # Run sender and receiver concurrently
+        sender_task = asyncio.create_task(send_text())
+
+        response_format = config.get("response_format", "wav")
+        word_timestamps = bool(config.get("word_timestamps", False))
+        current_sentence_index = 0
+        current_sentence_text = ""
+        current_chunks: list[bytes] = []
+        current_timestamps: list[dict] = []
+        # Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
+        # the sentence is a failure only if no frame ever carried timestamps.
+        alignment_frame_seen = False
+
+        try:
+            while True:
+                message = await ws.recv()
+
+                if isinstance(message, bytes):
+                    current_chunks.append(message)
+                    print(f"  Received audio chunk for sentence {current_sentence_index}: {len(message)} bytes")
+                else:
+                    # JSON frame
+                    msg = json.loads(message)
+                    msg_type = msg.get("type")
+
+                    if msg_type == "audio.start":
+                        current_sentence_index = msg["sentence_index"]
+                        current_sentence_text = msg.get("sentence_text", "")
+                        current_chunks = []
+                        current_timestamps = []
+                        alignment_frame_seen = False
+                        print(f"  [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}")
+                    elif msg_type == "audio.chunk":
+                        audio = base64.b64decode(msg["audio_b64"])
+                        current_chunks.append(audio)
+                        chunk_timestamps = msg.get("timestamps")
+                        # Sentence-level alignment sends a trailing timestamp-only
+                        # frame with empty audio; don't write an empty chunk file.
+                        if save_chunks and audio:
+                            chunk_base = os.path.join(
+                                output_dir,
+                                f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}",
+                            )
+                            chunk_pcm = f"{chunk_base}.pcm"
+                            chunk_wav = f"{chunk_base}.wav"
+                            chunk_json = f"{chunk_base}_timestamps.json"
+                            with open(chunk_pcm, "wb") as f:
+                                f.write(audio)
+                            save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
+                            with open(chunk_json, "w", encoding="utf-8") as f:
+                                json.dump(
+                                    {
+                                        "sentence_index": msg["sentence_index"],
+                                        "chunk_id": msg["chunk_id"],
+                                        "chunk_start_ms": msg.get("chunk_start_ms"),
+                                        "chunk_end_ms": msg.get("chunk_end_ms"),
+                                        "sentence_text": current_sentence_text,
+                                        "covered_text": timestamp_words_text(chunk_timestamps),
+                                        "timestamps": chunk_timestamps,
+                                    },
+                                    f,
+                                    ensure_ascii=False,
+                                    indent=2,
+                                )
+                        if chunk_timestamps is None:
+                            print(
+                                f"  [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
+                                f"{len(audio)} bytes, timestamps unavailable"
+                            )
+                        else:
+                            alignment_frame_seen = True
+                            current_timestamps.extend(chunk_timestamps)
+                            print(
+                                f"  [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
+                                f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
+                            )
+                        if save_chunks and audio:
+                            print(f"    saved chunk -> {chunk_wav}")
+                    elif msg_type == "audio.done":
+                        filename = save_audio_file(
+                            output_dir,
+                            msg["sentence_index"],
+                            response_format,
+                            current_chunks,
+                        )
+                        if word_timestamps:
+                            # `null` on failure, else the accumulated list (`[]` for silence).
+                            sentence_timestamps = current_timestamps if alignment_frame_seen else None
+                            ts_filename = os.path.join(
+                                output_dir,
+                                f"sentence_{msg['sentence_index']:03d}_timestamps.json",
+                            )
+                            with open(ts_filename, "w", encoding="utf-8") as f:
+                                json.dump(
+                                    {
+                                        "sentence_index": msg["sentence_index"],
+                                        "sentence_text": current_sentence_text,
+                                        "covered_text": timestamp_words_text(sentence_timestamps),
+                                        "timestamps": sentence_timestamps,
+                                    },
+                                    f,
+                                    ensure_ascii=False,
+                                    indent=2,
+                                )
+                        print(
+                            f"  [sentence {msg['sentence_index']}] Done"
+                            f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
+                            f" error={msg.get('error', False)}"
+                            f" -> {filename}"
+                        )
+                        if word_timestamps:
+                            print(f"  [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}")
+                        current_chunks = []
+                        current_timestamps = []
+                        alignment_frame_seen = False
+                    elif msg_type == "session.done":
+                        print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
+                        break
+                    elif msg_type == "error":
+                        print(f"  ERROR: {msg['message']}")
+                    else:
+                        print(f"  Unknown message: {msg}")
+        finally:
+            sender_task.cancel()
+            try:
+                await sender_task
+            except asyncio.CancelledError:
+                pass  # Task cancellation is expected during shutdown

    print(f"\nAudio files saved to: {output_dir}/")

@@ -325,8 +286,7 @@ def main():
    parser.add_argument(
        "--text",
        required=True,
-        action="append",
-        help="Text to synthesize; repeat to send several utterances over one connection",
+        help="Text to synthesize",
    )
    parser.add_argument(
        "--output-dir",
@@ -425,7 +385,7 @@ def main():
    asyncio.run(
        stream_tts(
            url=args.url,
-            texts=args.text,
+            text=args.text,
            config=config,
            output_dir=args.output_dir,
            simulate_stt=args.simulate_stt,
@@ -71,12 +71,6 @@ plugins:
  - search
  - autorefs
  - awesome-nav
-  - redirects:
-      redirect_maps:
-        user_guide/diffusion/request_batching.md: user_guide/diffusion/execution_modes.md
-        user_guide/diffusion/step_execution.md: user_guide/diffusion/execution_modes.md
-        design/feature/diffusion_request_level_batching.md: design/feature/diffusion_continuous_batching.md
-        design/feature/diffusion_step_execution.md: design/feature/diffusion_continuous_batching.md
  - glightbox
  - git-revision-date-localized:
      # exclude files
@@ -70,11 +70,11 @@ dev = [
    "FlagEmbedding==1.4.0", # for hunyuan_image3 tests
    "kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
    "s3tokenizer==0.3.0", # for tts test
-    "step-audio2==1.0.0", # Step-Audio2 / MiniCPM-o 4.5 Token2Wav (flashcosyvoice / hyperpyyaml)
+    "step-audio2==1.0.0", # Step-Audio2 Token2Wav (flashcosyvoice / hyperpyyaml)
+    "stepaudio2-minicpmo", # for MiniCPM-o 4.5 talker vocoder (stepaudio2-minicpmo Token2wav)
 ]

 demo = [
-    "gradio>=6.7.0",
    "opencv-python>=4.12.0.88",  # cli/stream_client frame reading
    "requests>=2.28.0",  # cli/stream_client HTTP
    "cn2an>=0.5.22",  # Chinese number normalization for TTS (e.g. "1次" -> "一次")
@@ -95,12 +95,6 @@ quack = [
    "quack-kernels>=0.3.11",
 ]

-# Optional Blackwell-native FlashAttention-4 backend. Keep this opt-in because
-# the CuTe/CUTLASS dependency is CUDA 13 and Blackwell specific.
-fa4 = [
-    "flash-attn-4[cu13]==4.0.0b18",
-]
-
 docs = [
    "mkdocs>=1.5.0",
    "mkdocs-api-autonav",
@@ -108,7 +102,6 @@ docs = [
    "mkdocstrings-python",
    "mkdocs-gen-files",
    "mkdocs-awesome-nav",
-    "mkdocs-redirects",
    "mkdocs-glightbox",
    "mkdocs-git-revision-date-localized-plugin",
    "mkdocs-minify-plugin",
@@ -0,0 +1,138 @@
+# Realtime Video Prompt Interaction Protocol
+
+This document specifies the prompt-update subset of the realtime video
+interaction protocol exposed by:
+
+```text
+WebSocket /v1/realtime/video
+```
+
+A WebSocket connection contains exactly one video-generation request. Clients
+therefore do not send a request or session ID with an interaction. The server
+assigns a `request_id` in `video.start` and includes it in subsequent
+server-to-client messages.
+
+## Interaction Request
+
+After `video.start`, a client may submit a prompt update:
+
+```json
+{
+  "type": "session.interaction",
+  "interaction": {
+    "event_id": "ui-update-42",
+    "event": {
+      "prompt": "A snowy street illuminated by warm lanterns"
+    },
+    "transition_chunks": 3
+  }
+}
+```
+
+`interaction` has the following fields:
+
+- `event_id` (optional string): session-unique idempotency key. The
+  server preserves a client-provided value or generates a UUID when omitted.
+- `event.prompt` (required string): non-empty target prompt.
+  over which the prompt reaches its target.
+- `transition_chunks` (optional non-negative integer): model-generation chunks
+  over which the prompt reaches its target. If omitted, the model's default transition is used.
+
+Note: Although `["interaction"]["event"]` only contains `prompt`, still consider using `OmniTextPrompt` for this part (with total=false) so that other fields will be supported in the future. Currently, `vllm_omni/diffusion/worker/diffusion_model_runner.py` already takes care of enforcing subset format, routing to meaningful input handing routines, and raising `NotImplementedError`. Only need to adapt this design to the input data structure above.
+
+### Transition Semantics
+
+Prompt transitions are target updates and begin at the next eligible model
+generation boundary after the interaction is processed.
+
+- `transition_chunks = 0` is a hard update: the first affected generation
+  chunk uses the target prompt.
+- `transition_chunks = N`, where `N > 0`, reaches the target in the `N`th
+  affected generation chunk. For example, a three-chunk linear transition uses
+  progress values `1/3`, `2/3`, and `1`.
+
+### Idempotency and Replacement
+
+Within a session:
+
+- repeating an `event_id` with the same payload is a retry and must not
+  apply the update twice;
+- repeating an `event_id` with a different payload is rejected with
+  `event_id_conflict`;
+- a newer prompt target supersedes any older pending prompt target;
+- when a new target interrupts an active transition, its transition starts
+  from the prompt state resolved at its effective boundary.
+
+## Queue Acknowledgement
+
+The server quickly acknowledges admission to the interaction queue:
+
+```json
+{
+  "type": "session.interaction.queued",
+  "request_id": "video_stream-...",
+  "event_id": "ui-update-42"
+}
+```
+
+`queued` confirms syntax validation, idempotency handling, and queue admission.
+It does not claim that the model has scheduled or applied the update. A later
+failure is reported as a correlated error:
+
+```json
+{
+  "type": "error",
+  "request_id": "video_stream-...",
+  "event_id": "ui-update-42",
+  "code": "interaction_failed",
+  "message": "The active model does not support prompt updates"
+}
+```
+
+## Video Chunk Metadata
+
+Each emitted binary payload is preceded by a JSON metadata message. The
+metadata message and its binary payload form one ordered pair and must not be
+interleaved with another outbound message.
+
+```json
+{
+  "type": "video.chunk_metadata",
+  "request_id": "video_stream-...",
+  "kind": "media",
+  "transport_chunk_index": 7,
+  "generation_chunk_index": 6,
+  "num_frames": 9,
+  "byte_length": 184320,
+  "started_event_ids": ["ui-update-42"],
+  "active_event_ids": ["ui-update-42"],
+  "completed_event_ids": []
+}
+```
+
+- `transport_chunk_index` identifies the binary WebSocket payload.
+- `generation_chunk_index` identifies the model-generation chunk represented
+  by the payload.
+- `started_event_ids` lists interactions that first affect this
+  generation chunk.
+- `active_event_ids` lists interactions contributing to this generation
+  chunk, including an ongoing transition.
+- `completed_event_ids` lists interactions that reach their target in
+  this generation chunk.
+
+Encoder trailer bytes are emitted as:
+
+```json
+{
+  "type": "video.chunk_metadata",
+  "request_id": "video_stream-...",
+  "kind": "trailer",
+  "transport_chunk_index": 8,
+  "generation_chunk_index": null,
+  "num_frames": 0,
+  "byte_length": 824,
+  "started_event_ids": [],
+  "active_event_ids": [],
+  "completed_event_ids": []
+}
+```
@@ -168,7 +168,7 @@ the model + orchestrator + WebUI together — point `WEBUI_DIR` at your clone.
 python examples/online_serving/joyvl_interaction/cli/run_cli_demo.py \
  path/to/video.mp4 --query "Alert me if a fire breaks out"

-pytest tests/e2e/features/fullduplex   # framework + JoyVL unit tests
+pytest tests/fullduplex   # framework + JoyVL unit tests
 ```

 ## Testing with an RTSP stream (optional)
@@ -165,40 +165,6 @@ Pass these fields in online `extra_params` or offline `--extra-body`:
 --extra-body '{"video_cfg_scale":3.0,"audio_cfg_scale":7.0}'
 ```

-### Guidance Parallelism
-
-For LTX, `cfg_parallel_size` is the number of ranks used to execute the
-complete guidance plan in parallel. Despite the legacy `cfg` name, it covers
-text CFG, STG, and cross-modality guidance passes; guidance rescaling is
-applied after all pass predictions are gathered.
-
-The default one-stage recipe has four Transformer passes per denoise step:
-`cond`, `uncond`, `ptb` (STG), and `mod` (cross-modality). The useful balanced
-configurations are therefore:
-
-| `--cfg-parallel-size` | Passes per rank | Guidance-slot utilization | Notes |
-|---:|---:|---:|---|
-| `1` | 4 | 100% | Single-rank fused guidance batch |
-| `2` | 2 | 100% | Recommended two-rank configuration |
-| `4` | 1 | 100% | One guidance pass per rank |
-
-Other positive sizes are accepted. When the pass count is not divisible by
-`cfg_parallel_size`, ranks are padded to an equal number of execution slots
-and LTX emits a warning with the expected utilization. For example, four
-passes on three ranks use six slots and have 66.7% guidance-slot utilization.
-
-Start an LTX-2.3 server with two-way guidance parallelism:
-
-```bash
-vllm serve diffusers/LTX-2.3-Diffusers --omni \
-  --cfg-parallel-size 2 --stage-init-timeout 600
-```
-
-The total device count is the product of `cfg_parallel_size` and the other
-configured parallel dimensions. Distilled pipelines use positive-only
-guidance, so `cfg_parallel_size > 1` does not provide guidance speedup for
-them.
-
 ### Python API

 Normal per-request values belong in `OmniDiffusionSamplingParams`. Pass the
@@ -307,8 +273,8 @@ bundled offline CLI do not currently expose `sigmas`.
 - Fused one-stage requests must resolve to identical LTX guidance and sigma
  schedules. Keep concurrent requests guidance-homogeneous or use
  `--max-num-seqs 1`.
- `--cfg-parallel-size` shards the complete LTX guidance plan, including STG,
-  modality guidance, and rescale-compatible prediction gathering.
+- `--cfg-parallel-size 2` supports only the CFG-only plan: disable STG and
+  modality guidance and set both rescale fields to `0.0`.

 ## Operational Notes

@@ -332,6 +298,6 @@ bundled offline CLI do not currently expose `sigmas`.
 - <https://huggingface.co/Lightricks/LTX-2.3>
 - <https://huggingface.co/diffusers/LTX-2.3-Diffusers>
 - [Online video generation](../../docs/user_guide/examples/online_serving/text_to_video.md)
- [Diffusion execution modes](../../docs/user_guide/diffusion/execution_modes.md)
+- [Request batching](../../docs/user_guide/diffusion/request_batching.md)
 - [T2V offline example](../../examples/offline_inference/text_to_video/text_to_video.md)
 - [I2V offline example](../../examples/offline_inference/image_to_video/README.md)
@@ -1,61 +0,0 @@
-# MiniMax-H3 on RTX 5090
-
-This recipe uses BF16 weights, tiled VAE decode, tensor parallelism where a
-second GPU is available, and distributed layerwise offload (DLO). It is a
-memory-first serving configuration; lower resident counts reduce HBM use and
-increase CPU-to-GPU transfer time.
-
-## Capacity requirements
-
-| Resource | One RTX 5090 | Two RTX 5090s |
-| --- | ---: | ---: |
-| GPU HBM | 32 GiB | 32 GiB per GPU |
-| Checkpoint storage | 135 GiB per partition | 135 GiB per partition |
-| Available system RAM | 200 GiB minimum | 200 GiB minimum |
-| Recommended system RAM | 384 GiB | 384 GiB |
-
-`FL2VA` and `Ref2VA` are separate 135 GiB checkpoint partitions. Start one
-server at a time. DLO keeps rank-local weights in pinned host memory; increasing
-`--dlo-resident-layers` improves latency but does **not** reduce host RAM in the
-current implementation because resident layers retain pinned CPU master copies.
-
-> **Modular H3:** after #5720 lands, preserve this recipe's one-partition
-> behavior with `--task-type fl2va` or `--task-type ref2va`.
-
-## One RTX 5090: 1344x768, 5 seconds
-
-Use 12 resident DiT layers. A 50-step B300 allocation test with this exact
-single-rank topology peaked at 26.50 GiB; re-measure peak HBM on the target
-card before increasing the resident count.
-
-```bash
-CUDA_VISIBLE_DEVICES=0 vllm serve /path/to/MiniMax-H3/FL2VA \
-  --omni --trust-remote-code --host 0.0.0.0 --port 8000 \
-  --num-gpus 1 --tensor-parallel-size 1 --text-encoder-tp-size 1 \
-  --usp 1 --ring 1 --vae-patch-parallel-size 1 \
-  --vae-parallel-mode tile --vae-use-tiling \
-  --enable-distributed-layerwise-offload --dlo-no-use-allgather \
-  --dlo-resident-layers 12 --enforce-eager \
-  --diffusion-attention-backend CUDNN_ATTN
-```
-
-## Two RTX 5090s: 1344x768, 5 seconds
-
-Use TP2 and 20 resident DiT layers. The two-rank B300 capacity run peaked at
-27,726 MiB per rank for this shape and 50 steps. This is a memory/correctness
-proxy, not a consumer-GPU latency claim.
-
-```bash
-CUDA_VISIBLE_DEVICES=0,1 vllm serve /path/to/MiniMax-H3/FL2VA \
-  --omni --trust-remote-code --host 0.0.0.0 --port 8000 \
-  --num-gpus 2 --tensor-parallel-size 2 --text-encoder-tp-size 2 \
-  --usp 1 --ring 1 --vae-patch-parallel-size 2 \
-  --vae-parallel-mode tile --vae-use-tiling \
-  --enable-distributed-layerwise-offload --dlo-no-use-allgather \
-  --dlo-resident-layers 20 --enforce-eager \
-  --diffusion-attention-backend CUDNN_ATTN
-```
-
-For Ref2VA, stop the FL2VA server and restart the same command with
-`/path/to/MiniMax-H3/Ref2VA`. Ref2VA reference video count and prompt length
-can increase activation memory; begin with one request at a time.
@@ -1,147 +0,0 @@
-# MiniMax H3 on Moore Threads MUSA
-
-> Joint video and audio generation on Moore Threads GPUs
-
-## Summary
-
- Vendor: MiniMaxAI
- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
- Tasks: T2VA, FL2VA, and Ref2VA
- Mode: OpenAI-compatible `/v1/videos` HTTP serving
- Hardware: 4x Moore Threads MTT S5000 for the validated Ref2VA profile
- Maintainer: Community
-
-This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for MUSA environments.
-MiniMax H3 has two independently served checkpoint partitions:
-
- `FL2VA` serves text-to-video+audio (`t2va`) and
-  first-frame-to-video+audio (`fl2va`).
- `Ref2VA` serves image/audio or video-reference generation (`ref2va`).
-
-## Prerequisites
-
-### Checkpoint
-
-Only download the partition needed by the server. Downloading the whole model
-repository also retrieves duplicate shared components and the original
-top-level pipeline layout.
-
-For T2VA and FL2VA:
-
-```bash
-python -m pip install modelscope
-export MODEL_ROOT=/path/to/MiniMax-H3
-modelscope download MiniMax/MiniMax-H3 \
-  --local_dir "${MODEL_ROOT}" \
-  --max-workers 16 \
-  --include 'FL2VA/**'
-```
-
-For Ref2VA, download `Ref2VA/**` instead.
-
-### Environment
-
-Install a compatible PyTorch/torch-musa, torchada, MATE, and vLLM-MUSA
-stack before installing vLLM-Omni. Importing vLLM-Omni should report that the
-`musa` vLLM platform plugin is active.
-
-Install vLLM-Omni from a checkout containing MiniMax H3 support:
-
-```bash
-python -m pip install -e .
-```
-
-MiniMax H3 reference inputs and MP4 output require `soundfile`, `ffmpeg`, and
-`ffprobe`. The Python dependency is installed by vLLM-Omni; install the two
-executables with the operating system package manager and verify them:
-
-```bash
-python -c 'import soundfile; print(soundfile.__version__)'
-ffmpeg -version
-ffprobe -version
-```
-
-When torchaudio cannot load TorchCodec, vLLM-Omni automatically falls back to
-soundfile. Formats that libsndfile cannot read are demuxed through ffmpeg.
-
-## Start a server
-
-One server loads one checkpoint partition. Set `MODEL` to `FL2VA` for T2VA
-and FL2VA, or to `Ref2VA` for Ref2VA.
-
-The validated Ref2VA configuration uses tensor parallelism across four MTT
-S5000 GPUs and offloads inactive model components to CPU:
-
-```bash
-export MODEL="${MODEL_ROOT}/FL2VA"
-export PORT=8091
-
-MUSA_VISIBLE_DEVICES=0,1,2,3 \
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 4 \
-  --tensor-parallel-size 4 \
-  --vae-patch-parallel-size 4 \
-  --enable-cpu-offload \
-  --vae-use-tiling \
-  --diffusion-attention-backend FLASH_ATTN
-```
-
-On MUSA, `FLASH_ATTN` selects the MATE/FlashAttention-3 implementation. Do
-not install the CUDA-only `fa4` optional dependency.
-
-For request examples and parameter definitions, see
-[MiniMax-H3.md](MiniMax-H3.md#http-api-examples).
-
-## MUSA validation
-
-The following weight-independent gates have been validated on one MTT S5000:
-
- packed variable-length attention selects MATE FlashAttention-3 and returns
-  finite BF16 output;
- MiniMax H3 RoPE casts CPU FP64 position metadata to FP32 before device
-  arithmetic and returns finite output;
- the video-VAE seeded RNG context is deterministic on MUSA and restores both
-  CPU and MUSA RNG states;
- the soundfile fallback preserves stereo channel layout, FP32 samples, and
-  the native sample rate when torchaudio/TorchCodec is unavailable.
-
-Ref2VA image+audio generation was also validated end to end on one MTT S5000
-with CPU offload and VAE tiling. The smoke used a 448x256 reference image, a
-four-second 32 kHz stereo audio reference, two denoising steps, and seed 42.
-The synchronous HTTP request completed in 50.6 seconds and returned an MP4
-with the following decoded properties:
-
- 107 video frames at 448x256 and 24 FPS;
- non-static video (`pixel_std=13.07`, `temporal_delta=0.49`);
- finite stereo audio at 32 kHz (`audio_rms=0.0458`).
-
-Official Ref2VA video-reference serving was validated with the expanded
-MiniMax prompt at 1344x768, 5 seconds, seed 0, and 50 steps. A true TP4 run on
-four MTT S5000 GPUs returned HTTP 200 in 638.6 seconds and produced 124 H.264
-frames at 24 FPS with AAC stereo audio at 32 kHz.
-
-Keep `--vae-use-tiling` enabled for this serving profile.
-
-## Known limitations
-
- Keep Ring Attention at degree 1. The current Ring path does not preserve
-  MiniMax H3 packed padding boundaries.
- H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
- Each server process loads only one checkpoint partition.
- H3 currently executes one generation request per diffusion batch.
- FP8 quantization has not been enabled for MiniMax H3.
- MP3, M4A, MP4, and reference-video audio fallback requires `ffmpeg` on
-  `PATH`; WAV inputs can be read directly through soundfile.
-
-## Additional resources
-
- [MiniMax-H3.md](MiniMax-H3.md) — full model and API guide
- [MiniMax-H3-NPU.md](MiniMax-H3-NPU.md) — Ascend NPU deployment guide
- [Supported models](../../docs/models/supported_models.md)
- [Video API](../../docs/serving/videos_api.md)
@@ -1,162 +0,0 @@
-# MiniMax H3 on Ascend NPU
-
-> Joint video and audio generation with text, first/last keyframes, and mixed
-> image/video/audio references — Ascend NPU deployment guide
-
-## Summary
-
- Vendor: MiniMaxAI
- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
- Tasks: T2VA, FL2VA, and Ref2VA
- Mode: OpenAI-compatible `/v1/videos` HTTP serving
- Hardware: Atlas 800I A3
- Maintainer: Community
-
-This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for Ascend NPU
-environments. Differences from the GPU path:
-
- Runs on a CPU-only (aarch64) PyTorch build with `torch_npu`; no CUDA
-  runtime is present.
- Audio loading does **not** require TorchCodec (whose aarch64 wheels are
-  built against CUDA torch and fail to load on CPU-only builds). vLLM-Omni
-  automatically falls back to soundfile / ffmpeg for wav/mp3/m4a/mp4 audio
-  inputs.
-
-## Prerequisites
-
-### Checkpoint
-
-Same as the GPU recipe — Hugging Face access approval is required:
-
-```bash
-hf auth login
-export MODEL_ROOT=/path/to/MiniMax-H3
-hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL_ROOT}"
-```
-
-### Environment
-
- Ascend driver & firmware: 25.5.1
- CANN toolkit: 9.0.1
- Python: 3.12
- PyTorch: 2.10.0+cpu
- torch_npu: 2.10.0.post2
- Install vLLM-Omni from a checkout with MiniMax H3 support:
-
-```bash
-uv venv
-source .venv/bin/activate
-uv pip install -e .
-```
-
-Install the **mindie-sd** operator library to enable Ascend-optimized fused
-operators (`adalayernorm`, etc.):
-
-```bash
-git clone https://gitcode.com/Ascend/MindIE-SD.git && cd MindIE-SD
-
-# Comment out the tik_ops build step (not needed for this use case)
-sed -i 's|^\(\s*\)source ${current_script_dir}/build_tik_ops.sh|\1# source ${current_script_dir}/build_tik_ops.sh|' build/build_ops.sh
-
-python setup.py bdist_wheel
-cd dist
-pip install mindiesd-*.whl
-```
-
- `ffmpeg` and `ffprobe` must be available on `PATH`. They are used for
-  reference-video preparation and MP4 output.
- Reference-video decoding uses `decord` when available and falls back to
-  PyAV otherwise.
- Audio inputs do not require TorchCodec: wav/mp3/m4a/mp4 files are loaded
-  at native sample rate through the soundfile / ffmpeg fallback.
-
-## Start a server
-
-One server loads one checkpoint partition. Set `MODEL` to `FL2VA` for T2VA
-and FL2VA requests, or to `Ref2VA` for Ref2VA requests.
-
-### Multi-NPU: 768P validated configuration
-
-Validated on eight NPUs of an Atlas 800I A3 server with Ulysses sequence
-parallelism degree 8, text-encoder tensor parallelism degree 8, native tiled
-VAE patch parallelism degree 8, and layerwise offload:
-
-```bash
-export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
-export PORT=9098
-export MODEL="${MODEL_ROOT}/FL2VA"
-export VLLM_WORKER_MULTIPROC_METHOD=spawn
-export VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800
-export PYTHONDONTWRITEBYTECODE=1
-export HF_HUB_OFFLINE=1
-export TRANSFORMERS_OFFLINE=1
-
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 8 \
-  --usp 8 \
-  --ring 1 \
-  --text-encoder-tp-size 8 \
-  --enable-layerwise-offload \
-  --vae-parallel-mode tile \
-  --vae-use-tiling \
-  --vae-patch-parallel-size 8 \
-  --diffusion-attention-backend FLASH_ATTN
-```
-
-Do not add `--enforce-eager`. The first request includes regional
-compilation; warm the server once before measuring steady-state latency.
-H3 is CFG-distilled, so `--cfg-parallel-size` must remain 1.
-
-To serve Ref2VA, stop the FL2VA server and restart with:
-
-```bash
-export MODEL="${MODEL_ROOT}/Ref2VA"
-```
-
-## HTTP API examples
-
-The request format is identical to the GPU recipe; see
-[MiniMax-H3.md § HTTP API examples](MiniMax-H3.md#http-api-examples).
-Use the validated 768P shapes (e.g. `width=1344 height=768`) on NPU.
-
-## Key parameters
-
-Same as the GPU recipe; see
-[MiniMax-H3.md § Key parameters](MiniMax-H3.md#key-parameters).
-The validated resolution on NPU is 768P (e.g. 1344x768).
-
-## Validated NPU evidence
-
-Measured on an Atlas 800I A3 server (8x NPU) with CANN 9.0.1,
-PyTorch 2.10.0+cpu, and torch_npu 2.10.0.post2, using the multi-NPU
-configuration above:
-
-| Workload | Configuration |
-|----------|---------------|
-| T2VA, 209 frames, 1344x768 | TE TP8, layerwise offload, Ulysses 8, VPP8 tile, regional compile |
-| Ref2VA (prompt + video), 124 frames, 1344x768 | TE TP8, layerwise offload, Ulysses 8, VPP8 tile, regional compile |
-
-These measurements describe the validated shapes rather than a general
-throughput guarantee.
-
-## Known limitations
-
- Each server process loads only one checkpoint partition.
- H3 currently executes one generation request per diffusion batch.
- The first regional-compile request is a warmup and should not be included
-  in steady-state performance measurements.
- The official H3 input matrix and media limits are documented in the [GPU
-  recipe](MiniMax-H3.md#official-input-matrix-and-limits); this NPU path uses
-  the same HTTP request contract.
- VAE patch parallelism requires size 1 or the full DiT group size and
-  supports the H3 native `tile` mode only.
-
-## Additional resources
-
- [MiniMax-H3.md](MiniMax-H3.md) — full GPU guide
- [Supported models](../../docs/models/supported_models.md)
- [Video API](../../docs/serving/videos_api.md)
@@ -1,568 +0,0 @@
-# MiniMax H3
-
-> Joint video and audio generation with text, first/last keyframes, and
-> mixed image/video/audio references
-
-## Summary
-
- Vendor: MiniMaxAI
- Model: [`MiniMaxAI/MiniMax-H3`](https://huggingface.co/MiniMaxAI/MiniMax-H3)
- Tasks: T2VA, FL2VA, and Ref2VA
- Mode: OpenAI-compatible `/v1/videos` HTTP serving
- Maintainer: Community
-
-MiniMax H3 is a CFG-distilled joint video/audio diffusion transformer. Its
-checkpoint has two independently served partitions:
-
- `FL2VA`: text-to-video+audio (`t2va`) and first-frame-to-video+audio
-  (`fl2va`)
- `Ref2VA`: up to 9 images, 3 videos, and 3 audio references in supported
-  image/video/audio combinations (`ref2va`; audio-only is rejected)
-
-The generated MP4 contains H.264 video and synchronized stereo audio.
-
-## Prerequisites
-
-The checkpoint requires Hugging Face access approval. Download it and point
-`MODEL_ROOT` at the directory containing the `FL2VA` and `Ref2VA`
-subdirectories:
-
-```bash
-hf auth login
-export MODEL_ROOT=/path/to/MiniMax-H3
-hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL_ROOT}"
-```
-
-Install vLLM-Omni from the checkout containing MiniMax H3 support. The
-two-GPU RTX 5090/4090 profiles use cuDNN attention and do not need
-FlashAttention-4. Install the optional dependency only for the four-GPU
-B300/GB200 `FLASH_ATTN` profile:
-
-```bash
-uv venv
-source .venv/bin/activate
-uv pip install -e '.[fa4]'
-```
-
-`ffmpeg` and `ffprobe` must be available on `PATH`. They are used for
-reference-video preparation and MP4 output.
-
-## Start a server
-
-One server loads one checkpoint partition. Set `MODEL` to `FL2VA` for T2VA
-and FL2VA requests, or to `Ref2VA` for either Ref2VA request.
-
-### Memory and storage requirements
-
-Treat GPU HBM, host RAM, and checkpoint storage as separate requirements. Each
-H3 checkpoint partition (`FL2VA` or `Ref2VA`) contains about **134 GiB** of
-BF16 safetensors (about **135 GiB** on disk). Keeping both partitions locally
-therefore needs roughly **270 GiB** of model storage, although the examples
-serve only one partition at a time.
-
-CPU offload and distributed layerwise offload reduce GPU residency; they do
-not make the model weights disappear. With `--dlo-no-use-allgather`, each
-worker retains its standard-loader rank-local weights in host memory, including
-pinned CPU buffers used for H2D streaming. Use at least **200 GiB available
-system RAM** before starting the two-GPU recipe; a **384 GiB host is
-recommended** to leave room for the OS, CUDA/PyTorch allocations, request
-inputs, and filesystem cache. Do not run the FL2VA and Ref2VA servers at the
-same time on a host sized for this minimum.
-
-The consumer-GPU profiles below are HBM budgets only. They still require the
-host-RAM budget above.
-
-### Single GPU: accuracy and memory first
-
-The single-GPU configuration uses model-level CPU offload.
-This matches the accuracy-qualified reference path and prevents the Qwen3-VL
-encoder and DiT from being resident on the GPU at the same time.
-
-```bash
-export MODEL="${MODEL_ROOT}/FL2VA"
-export PORT=8091
-
-CUDA_VISIBLE_DEVICES=0 \
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 1 \
-  --enable-cpu-offload \
-  --diffusion-attention-backend FLASH_ATTN
-```
-
-Use a GPU with enough memory for the active H3 component and enough system RAM
-for the offloaded components. CPU offload reduces GPU memory pressure but adds
-PCIe/NVLink transfer latency.
-
-### Two 24/32 GB GPUs: TP2 distributed layerwise offload
-
-For two PCIe consumer GPUs, combine TP2 with distributed layerwise offload
-(DLO). The standard loader first creates the rank-local TP shard. DLO keeps
-that shard in pinned host memory and streams the 30 tail DiT blocks through a
-shared two-buffer window without a DP AllGather. The first 20 DiT blocks are
-copied to the GPUs once per denoise stage, reused by every sampling step, and
-released before VAE decode so the decoder can reuse their HBM.
-
-```bash
-export MODEL="${MODEL_ROOT}/FL2VA"
-export PORT=8091
-
-CUDA_VISIBLE_DEVICES=0,1 \
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 2 \
-  --tensor-parallel-size 2 \
-  --usp 1 \
-  --ring 1 \
-  --text-encoder-tp-size 2 \
-  --vae-patch-parallel-size 2 \
-  --vae-parallel-mode tile \
-  --vae-use-tiling \
-  --enable-distributed-layerwise-offload \
-  --dlo-no-use-allgather \
-  --dlo-resident-layers 20 \
-  --enforce-eager \
-  --diffusion-attention-backend CUDNN_ATTN
-```
-
-Use the profile that matches the per-GPU memory capacity:
-
-| Profile | GPUs | Starting shape | Resident DiT blocks | Attention | Execution | Status |
-|---|---:|---:|---:|---|---|---|
-| `rtx5090` | 2 x 32 GB | 1344x768 | 20 | cuDNN attention | eager | Target-hardware validated |
-| `rtx4090` | 2 x 24 GB | 1024x576 | 12 | cuDNN attention | eager | Capacity-proxy starting point |
-
-This topology uses all available parallel capacity: TP2 shards both the DiT
-and text encoder, `--dlo-no-use-allgather` streams each rank's local TP shard
-without reconstructing full blocks, and VAE patch parallelism splits tiled
-decode across both GPUs. cuDNN attention is selected explicitly for the RTX
-consumer path; the server stays eager to avoid an unqualified compile path.
-
-The resident count changes placement and transfer frequency only; it does not
-quantize or change the BF16/FP32 denoise math. Re-measure peak memory before
-increasing it on a different request shape.
-
-### RTX 5090 target-hardware validation
-
-At vLLM-Omni commit `ae6577ea`, one full 50-step T2VA request completed on
-2 x RTX 5090 without OOM:
-
-| Shape | Frames | Client E2E | Sampled peak/GPU | Output validation |
-|---:|---:|---:|---:|---|
-| 1344x768 | 124 at 24 FPS | 8 min 38 s | approximately 22.6 GiB | H.264 video + 32 kHz stereo AAC; full `ffmpeg` decode passed |
-
-This is a single end-to-end validation run, not a warmed multi-run latency
-benchmark. The sampled `nvidia-smi` peak is also not a CUDA allocator
-high-water mark. The environment used vLLM 0.26.0, vLLM-Omni
-`0.26.1.dev14+gae6577ea`, and PyTorch 2.11.0+cu130. The
-[run record](https://github.com/lishunyang12/vllm-omni-rankings/blob/dcd06d7e83cb069842535918c0169ee9f3f29ba0/scripts/%E5%BE%AE%E4%BF%A1%E5%9B%BE%E7%89%87_20260805000034_86_237.png)
-captures the environment, output contract, elapsed time, and sampled peak.
-
-Before the target run, both profiles were exercised on two B300 ranks as an
-allocation and correctness proxy. At 1344x768, 124 frames, and 50 steps, the
-20-layer profile peaked at 27,726 MiB per rank. At 1024x576, the 12-layer
-profile peaked at 18,888 MiB per rank in a 5-step capacity run. The resident
-and fully streamed placements produced identical decoded video-frame and audio
-hashes for the same shape, step count, prompt, and seed. The B300 result does
-not establish RTX 4090 PCIe latency; treat the 4090 profile as a conservative
-starting point until it is measured on that GPU.
-
-To run T2VA, FL2VA, image+audio Ref2VA, and two-video Ref2VA in order, validate
-every MP4's H.264/AAC streams, and retain live server and GPU-memory logs:
-
-```bash
-RUN_ROOT=/path/to/run-root \
-MODEL_ROOT=/path/to/MiniMax-H3 \
-GPU_IDS=0,1 \
-PROFILE=rtx5090 \
-bash examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh
-```
-
-The script selects 20 resident layers for `PROFILE=rtx5090` and 12 for
-`PROFILE=rtx4090`; `DLO_RESIDENT_LAYERS=N` overrides either default.
-
-### Four GPUs: measured best-practice throughput
-
-The best validated four-GPU configuration on four NVIDIA B300 GPUs is:
-
- no CPU or layerwise offload;
- Ulysses sequence parallelism degree 4;
- native tiled VAE patch parallelism degree 4;
- regional `torch.compile` for the repeated DiT blocks;
- CuTe FlashAttention-4 through the `FLASH_ATTN` backend, with Ring and TP
-  left at 1.
-
-```bash
-export MODEL="${MODEL_ROOT}/FL2VA"
-export PORT=8091
-
-CUDA_VISIBLE_DEVICES=0,1,2,3 \
-FLASHINFER_DISABLE_VERSION_CHECK=1 \
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 4 \
-  --usp 4 \
-  --ring 1 \
-  --vae-patch-parallel-size 4 \
-  --vae-parallel-mode tile \
-  --vae-use-tiling \
-  --diffusion-attention-backend FLASH_ATTN
-```
-
-On Blackwell, `FLASH_ATTN` selects FA4. Confirm the server log contains
-`Using CuTe FlashAttention-4 on Blackwell` before recording measurements.
-
-Do not add `--enforce-eager` to this performance configuration. The first
-request includes regional compilation; warm the server once before measuring
-steady-state latency. H3 is CFG-distilled, so `--cfg-parallel-size` must remain
-1. The H3 VAE supports its native `tile` mode, not
-`spatial_shard_height` or `spatial_shard_width`.
-
-### Text encoder tensor parallelism
-
-The Qwen3-VL text encoder (~51.5 GB in BF16 for the retained 50 layers) is by
-default fully resident on the DiT main rank.  On multi-GPU no-offload runs that
-rank becomes the peak-memory hotspot.  Add `--text-encoder-tp-size N` to shard
-the encoder across the first `N` DiT ranks (the encoder is implemented with
-vLLM-style tensor-parallel layers and runs with distributed collectives over
-its own encoder process group):
-
-```bash
-export MODEL="${MODEL_ROOT}/FL2VA"
-export PORT=8091
-
-CUDA_VISIBLE_DEVICES=0,1,2,3 \
-FLASHINFER_DISABLE_VERSION_CHECK=1 \
-VLLM_WORKER_MULTIPROC_METHOD=spawn \
-VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \
-vllm serve "${MODEL}" \
-  --omni \
-  --host 0.0.0.0 \
-  --port "${PORT}" \
-  --trust-remote-code \
-  --num-gpus 4 \
-  --usp 4 \
-  --ring 1 \
-  --text-encoder-tp-size 4 \
-  --vae-patch-parallel-size 4 \
-  --vae-parallel-mode tile \
-  --vae-use-tiling \
-  --diffusion-attention-backend FLASH_ATTN
-```
-
-`N` must divide the Qwen3-VL head counts (64 attention heads / 8 KV heads), so
-valid values on a 4-GPU server are 1, 2, and 4 (1, 2, 4, 8 on 8 GPUs).  The
-encoder TP rank set is the first `N` DiT ranks; on 4 GPUs
-`--text-encoder-tp-size 4` shards the encoder 4-way, dropping the DiT main
-rank's no-offload peak by roughly `(N-1)/N` of the ~51.5 GB encoder while the
-other ranks each gain `~51.5/N` GB.  The encoder output remains identical to
-the reference path within bf16 rounding: every encoder rank all-reduces the
-row-parallel projections, so the full `[seq, 5120]` layer-50 hidden state is
-replicated on every rank.
-
-To serve Ref2VA, stop the FL2VA server and restart the same one- or four-GPU
-command with:
-
-```bash
-export MODEL="${MODEL_ROOT}/Ref2VA"
-```
-
-### Online FP8 quantization
-
-MiniMax H3 supports load-time FP8 quantization of the DiT. The checkpoint
-remains BF16 on disk; vLLM-Omni quantizes eligible weights while loading and
-uses dynamic activation scaling during inference. By default, attention and
-MLP linears in the token refiner and main DiT blocks, the condition
-projection, and all AdaLN projections use FP8. Patch, timestep, and final
-projections remain FP32; the text encoder and VAEs are unchanged.
-
-Add this option to an existing H3 server command:
-
-```bash
--quantization fp8
-```
-
-Use `ignored_layers` to keep any otherwise eligible linear in BF16. H3
-resolves the `transformer` component before constructing the DiT, so names do
-not start with `transformer.`. Entries are exact runtime linear prefixes; a
-parent name such as `blocks.0.attn` does not exclude its children.
-
-Eligible names are the `attn.qkv_proj`, `attn.out_proj`, `mlp.fc1`, and
-`mlp.fc2` children under `token_refiner.blocks.<0-1>` or `blocks.<0-49>`.
-The other eligible names are `condition_proj`,
-`blocks.<0-49>.adaln_proj.linear`, and `final_layer.adaln_proj.linear`.
-For example, keep the first main block's attention projections in BF16 with:
-
-```bash
--diffusion-quantization-config \
-  '{"transformer":{"method":"fp8","ignored_layers":["blocks.0.attn.qkv_proj","blocks.0.attn.out_proj"]}}'
-```
-
-The structured option replaces `--quantization fp8`. Online FP8 is currently
-incompatible with H3 layerwise offload because the offload path produces a
-weight stride rejected by the Cutlass FP8 kernel. Use resident FP8 with tensor
-parallelism and VAE tiling instead.
-
-## HTTP API examples
-
-The following requests use the synchronous endpoint so the returned body can
-be saved directly as an MP4. The asynchronous `POST /v1/videos` endpoint can
-also be used when job polling is preferred.
-
-All four tasks use 24 FPS, 50 sigma points, seed values from the validated
-workloads, and the checkpoint-reference video/audio flow shifts of 12 and 3.
-Decimal durations are passed through `extra_params`.
-
-Set the endpoint once:
-
-```bash
-export API_URL="http://127.0.0.1:${PORT}/v1/videos/sync"
-```
-
-### 1. T2VA: text to video and audio
-
-Run this request against the `FL2VA` partition:
-
-```bash
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=In a snowy blue-purple forest, Ori carefully walks past a sleeping giant; footsteps crunch in the snow while the creature breathes and softly snorts.' \
-  -F 'width=1344' \
-  -F 'height=768' \
-  -F 'aspect_ratio=16:9' \
-  -F 'fps=24' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=1101' \
-  -F 'extra_params={"task":"t2va","duration":8.7,"audio_flow_shift":3.0}' \
-  -o t2va.mp4
-```
-
-### 2. FL2VA: first frame to video and audio
-
-Run this request against the `FL2VA` partition. When width and height are
-omitted, H3 preserves the first-frame aspect ratio and uses a 768-pixel short
-edge.
-
-```bash
-export FIRST_FRAME=/path/to/fl2va_first_frame.png
-
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=A man stands beside a yellow car at night. The car drives away; he follows it with his eyes and begins singing sadly, with synchronized voice and city ambience.' \
-  -F 'fps=24' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=2101' \
-  -F 'extra_params={"task":"fl2va","duration":8.7,"audio_flow_shift":3.0}' \
-  -F "input_reference=@${FIRST_FRAME};type=image/png" \
-  -o fl2va.mp4
-```
-
-Use the same `FL2VA` partition for the official tail-keyframe forms. A single
-image with `frame_indices=[-1]` conditions the last frame; two ordered images
-with `frame_indices=[0,-1]` condition the first and last frames:
-
-```bash
-export LAST_FRAME=/path/to/fl2va_last_frame.png
-export FIRST_FRAME=/path/to/fl2va_first_frame.png
-
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=The subject moves naturally from the first image to the last image.' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=2102' \
-  -F 'extra_params={"task":"fl2va","duration":8.7,"frame_indices":[0,-1],"audio_flow_shift":3.0}' \
-  -F "input_references=@${FIRST_FRAME};type=image/png" \
-  -F "input_references=@${LAST_FRAME};type=image/png" \
-  -o fl2va_first_last.mp4
-```
-
-### 3. Ref2VA: image-only, image/audio, or mixed references
-
-Run these requests against the `Ref2VA` partition. Image-only Ref2VA omits
-`audio_reference`; adding one or more audio references is optional. The typed
-fields accept one object or an ordered JSON list. `audio_reference` accepts an
-HTTP(S) URL or a `data:` URL.
-
-```bash
-python -m http.server 8092 \
-  --bind 127.0.0.1 \
-  --directory /path/to/reference_assets
-```
-
-Then submit an image-only request from another terminal:
-
-```bash
-export REF_IMAGE=/path/to/reference_assets/ref2va_image.png
-
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=A white cat sits on a beige couch and slowly looks toward the camera.' \
-  -F 'aspect_ratio=adaptive' \
-  -F 'short_edge=768' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=3100' \
-  -F 'extra_params={"task":"ref2va","duration":8.0,"audio_flow_shift":3.0}' \
-  -F "input_reference=@${REF_IMAGE};type=image/png" \
-  -o ref2va_image_only.mp4
-```
-
-An image-plus-audio request is:
-
-```bash
-export REF_IMAGE=/path/to/reference_assets/ref2va_image.png
-export AUDIO_URL=http://127.0.0.1:8092/ref2va_audio.mp3
-
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=A white cat with black mustache and eyebrow markings sits on a beige couch, lip-syncing precisely to the complete reference audio before shifting from confusion to deadpan speechlessness.' \
-  -F 'width=1344' \
-  -F 'height=768' \
-  -F 'fps=24' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=3101' \
-  -F 'extra_params={"task":"ref2va","duration":15.0,"audio_flow_shift":3.0}' \
-  -F "input_reference=@${REF_IMAGE};type=image/png" \
-  -F "audio_reference={\"audio_url\":\"${AUDIO_URL}\"}" \
-  -o ref2va_image_audio.mp4
-```
-
-The requested duration should cover the complete audio. If `duration` is
-shorter, the reference soundtrack is truncated to the generated clip.
-
-### 4. Ref2VA: video, separate audio, and mixed references
-
-Run this request against the `Ref2VA` partition. Repeat the
-`input_references` multipart field once per source video. H3 consumes the
-videos in form order and preserves their original soundtracks during
-conditioning.
-
-```bash
-export SUBJECT_VIDEO=/path/to/green_screen_subject.mp4
-export BACKGROUND_VIDEO=/path/to/fairytale_background.mov
-
-curl -sS -X POST "${API_URL}" \
-  -F 'prompt=Remove the green screen background of Video 1 and replace it with the fairytale environment from Video 2. Match the background motion to the character actions and relight the character to fit the scene.' \
-  -F 'width=1344' \
-  -F 'height=768' \
-  -F 'fps=24' \
-  -F 'num_inference_steps=50' \
-  -F 'flow_shift=12' \
-  -F 'seed=3101' \
-  -F 'extra_params={"task":"ref2va","duration":15.0,"audio_flow_shift":3.0}' \
-  -F "input_references=@${SUBJECT_VIDEO};type=video/mp4" \
-  -F "input_references=@${BACKGROUND_VIDEO};type=video/quicktime" \
-  -o ref2va_video_video.mp4
-```
-
-The server stores uploaded references only for the lifetime of the request and
-deletes temporary files after generation. A video may use its embedded
-soundtrack, a separate `audio_reference`, or both. To send a mixed multipart
-request, repeat `input_references` for each image, video, or audio file; the
-server classifies them by MIME type and preserves the per-type order.
-
-Reference videos must be MP4/MOV with H.264/H.265 video, optional AAC/MP3
-audio, 2–15 seconds each, and at most 15 seconds combined. They may still be
-longer than the generated clip. Use `start_time_seconds` to select a
-synchronized segment; for multiple typed video references, pass one value per
-video in `extra_params.start_time_seconds`.
-
-Reference images accept JPG/JPEG, PNG, WEBP, HEIC, or HEIF up to 30 MiB. Standalone
-audio references accept WAV or MP3 up to 15 MiB, with 2–15 seconds per file and
-at most 15 seconds combined.
-
-## Official input matrix and limits
-
-| Task | Supported references | Limits |
-|------|----------------------|--------|
-| T2VA | text only | prompt must be non-empty |
-| FL2VA | first image, last image, or ordered first+last images | at most 2 images; `frame_indices` is `[0]`, `[-1]`, or `[0,-1]` |
-| Ref2VA | image-only, image+image, image+video, video+audio, and mixed image/video/audio | images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference |
-
-The H3 output contract is 4–15 seconds at 24 FPS, stereo 32 kHz audio, and a
-32-pixel canvas multiple. T2VA requires one named output ratio from `21:9`,
-`16:9`, `4:3`, `1:1`, `3:4`, or `9:16`. FL2VA always follows the first input
-image's ratio and ignores a generic `aspect_ratio` override. Ref2VA defaults to
-`16:9`; `adaptive` and SGLang's `auto` spelling are accepted aliases for that
-default. `short_edge` controls the 768-pixel canvas and must be `768`.
-`num_outputs_per_prompt` accepts 1–10 and derives each output seed as
-`seed + output_index`. The asynchronous endpoint returns all
-outputs; the synchronous raw-MP4 endpoint returns the first output when more
-than one is requested.
-
-## Key parameters
-
-| Parameter | Recommended value | Notes |
-|-----------|-------------------|-------|
-| `task` | `t2va`, `fl2va`, or `ref2va` | Passed in `extra_params`; must match the served partition |
-| `duration` | Workload-specific | Decimal seconds in `extra_params`; converted to H3-compatible frame count |
-| `fps` | `24` | H3 output FPS is fixed |
-| `num_inference_steps` | `50` | Matches the reference accuracy workloads |
-| `flow_shift` | `12` | Video sigma shift |
-| `audio_flow_shift` | `3` | Audio sigma shift, passed in `extra_params` |
-| `seed` | Task-specific | Use a fixed value for reproducibility |
-| `aspect_ratio` | Task-specific | T2VA requires a named ratio; FL2VA follows the input image; Ref2VA defaults to `16:9` |
-| `short_edge` | `768` | H3 shape policy requires exactly 768 when `width`/`height` are omitted |
-| `num_outputs_per_prompt` | `1` | 1–10; async API returns every output |
-| `start_time_seconds` | `0` | Reference-video segment start; use a list in `extra_params` for multiple videos |
-| `width`, `height` | Multiples of 32 | Output aspect ratio must be between 1:4 and 4:1 |
-
-## ComfyUI Frontend
-
-Users can also use a ComfyUI frontend to interact with a hosted MiniMax-H3 service. The ComfyUI frontend can run in a separate environment or machine. Refer to [vLLM-Omni ComfyUI Integration](../../docs/features/comfyui.md) for details.
-
-## Validated four-GPU evidence
-
-The four-GPU recommendation was measured on four NVIDIA B300 GPUs with one
-excluded warmup followed by three requests.
-
-| Workload | Configuration | Observed result |
-|----------|---------------|-----------------|
-| FL2VA, 209 frames, 1248x768 | no offload, U4, VPP4 tile, regional compile | 86.964 s mean HTTP client latency |
-| Two-video Ref2VA, 362 frames, 1344x768 | no offload, U4, VPP4 tile, regional compile | 784.394 s accounted model-stage mean |
-
-These measurements describe the validated shapes rather than a general
-throughput guarantee. Multi-video Ref2VA is much slower because the two
-reference videos expand both the Qwen3-VL vision sequence and the packed DiT
-attention sequence.
-
-## Validated FP8 evidence
-
-With eager DiT/text-encoder TP2 and VAE tiling, the 384x672, 107-frame,
-10-step quality case measured LPIPS 0.1156 (limit 0.20), PSNR 23.6316 dB,
-audio spectral cosine 0.9589 (minimum 0.80), and audio RMS ratio 0.9342. The
-resident per-GPU peak was 68.52 GiB for BF16 and 53.51 GiB for FP8, a 22%
-reduction.
-
-## Known limitations
-
- Each server process loads only one checkpoint partition.
- H3 currently executes one generation request per diffusion batch.
- The first regional-compile request is a warmup and should not be included in
-  steady-state performance measurements.
- Online FP8 is not compatible with layerwise offload.
- Image+audio Ref2VA accepts exactly one image and one audio reference.
- Video Ref2VA accepts one or more video files, but not an additional standalone
-  audio reference.
- VAE patch parallelism requires size 1 or the full DiT group size and supports
-  the H3 native `tile` mode only.
-
-## Additional resources
-
- [Supported models](../../docs/models/supported_models.md)
- [Video API](../../docs/serving/videos_api.md)
- [Diffusion parallelism](../../docs/user_guide/diffusion/parallelism/overview.md)