Revert "[Doc][Frontend][Model][VoxCPM2] Support instructions and per-request …"

This reverts commit 1069a144a8.

.buildkite/pipeline.yml

@@ -1,8 +1,9 @@
 # Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs if nightly-test
-# / main NIGHTLY so upload-nightly is not skipped together with test-ready/test-merge.
-#
-# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite).
+# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
+# exception in upload_pipeline_with_skip_ci.sh (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
+# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
+
+# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
 steps:
   - label: ":github: Resolve skip-ci & upload pipeline"
     key: upload-ci-pipeline
@@ -61,3 +62,15 @@ steps:
       - buildkite-agent pipeline upload .buildkite/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"
+
+  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
+  - label: "Upload Weekly Pipeline"
+    depends_on: image-build
+    key: upload-weekly-pipeline
+    if: >-
+      (build.branch == "main" && build.env("WEEKLY") == "1") ||
+      (build.branch != "main" && build.pull_request.labels includes "weekly-test")
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-weekly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -141,5 +141,5 @@ docker run \
     "${image_name}" \
     bash -c '
     set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
 '

.buildkite/test-amd-merge.yml

@@ -111,8 +111,8 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2

.buildkite/test-amd-ready.yaml

@@ -112,19 +112,6 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
-
-# NOTE: This test is not running any thing. It is skipped and deselected.
-# Currently it is = 1 skipped, 1 deselected, 17 warnings in 0.03s ======
-# - label: "Omni Model Test Qwen2-5-Omni"
-#   agent_pool: mi325_2
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py -m "core_model" --run-level "core_model"
-
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi325_2
 #   depends_on: amd-build

.buildkite/test-amd.yaml

@@ -88,7 +88,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
     - pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
 - label: "Omni Model Test Qwen3-Omni"

.buildkite/test-merge.yml

@@ -172,25 +172,6 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "Omni Model Test"
-    timeout_in_minutes: 30
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: "Qwen3-TTS CustomVoice E2E Test"
     depends_on: upload-merge-pipeline
     commands:
@@ -235,6 +216,28 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
+  # - label: "MOSS-TTS-Nano E2E Test"
+  #   depends_on: upload-merge-pipeline
+  #   commands:
+  #     - |
+  #       timeout 20m bash -c '
+  #         export VLLM_LOGGING_LEVEL=DEBUG
+  #         export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+  #         pytest -s -v tests/e2e/online_serving/test_moss_tts_nano.py tests/e2e/offline_inference/test_moss_tts_nano.py -m "advanced_model" --run-level "advanced_model"
+  #       '
+  #   agents:
+  #     queue: "gpu_1_queue"
+  #   plugins:
+  #     - docker#v5.2.0:
+  #         image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+  #         always-pull: true
+  #         propagate-environment: true
+  #         shm-size: "8gb"
+  #         environment:
+  #           - "HF_HOME=/fsx/hf_cache"
+  #         volumes:
+  #           - "/fsx/hf_cache:/fsx/hf_cache"
+
   - label: "Omni Model Test with H100"
     timeout_in_minutes: 30
     depends_on: upload-merge-pipeline
@@ -363,7 +366,7 @@ steps:
           set -e
           export VLLM_TEST_CLEAN_GPU_MEMORY=1
           export VLLM_IMAGE_FETCH_TIMEOUT=60
-          pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory"
+          pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model"
           pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model"
         '
     agents:

.buildkite/test-nightly.yml

@@ -10,7 +10,7 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
     steps:
-      - label: ":full_moon: Omni · Function Test"
+      - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
@@ -48,6 +48,25 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+      - label: ":full_moon: Omni · Function Test with L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+        agents:
+          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
       - label: ":full_moon: Omni · Doc Test with L4"
         timeout_in_minutes: 90
         commands:
@@ -205,7 +224,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" -k "not test_qwen2_5" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:

.buildkite/test-weekly.yml

@@ -0,0 +1,85 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - label: "Reliability Test - qwen3-omni"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  - label: "Reliability Test - wan22"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate

.claude/skills/add-diffusion-model/SKILL.md

@@ -544,6 +544,15 @@ Pipelines without `SupportsModuleOffload` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 
+### Step 12: Performance Profiling
+
+After verifying correctness and implementing parallelism/caching, profile the model's performance to identify bottlenecks and ensure optimal execution.
+
+See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.md#3-profiling-single-stage-diffusion) guide for detailed instructions on:
+1. Using the PyTorch profiler (`profiler: "torch"`) to capture detailed CPU/CUDA traces.
+2. Using Nsight Systems (`nsys`) with `profiler: "cuda"` for low-overhead CUDA traces.
+3. Controlling profiling via `omni.start_profile()` and `omni.stop_profile()`.
+
 ---
 
 ## Iterative Development Tips

benchmarks/build_dataset/seed_tts_design/en/meta.lst

@@ -0,0 +1,20 @@
+vd001|||The quick brown fox jumps over the lazy dog.|A warm, friendly female voice with a slight American Midwest accent, speaking at a moderate pace with natural inflection.
+vd002|||Welcome to the future of text-to-speech synthesis.|A deep, authoritative male news anchor voice, clear and professional with a measured cadence.
+vd003|||The sunset painted the sky in brilliant shades of orange and pink.|A gentle elderly female voice, soft and wise, with a slight Southern American accent.
+vd004|||Scientists have discovered a new species of deep-sea creature.|A young male voice with an Australian accent, curious and enthusiastic.
+vd005|||Breaking news: a major climate summit opens today in Geneva.|A crisp female newsreader voice, neutral accent, confident and precise.
+vd006|||In the beginning, there was darkness and silence across the universe.|A rich, dramatic bass male narrator voice, slow and deeply resonant.
+vd007|||Come closer, I have something important to tell you.|A soft, intimate female voice, slightly whispery, warm and gentle.
+vd008|||And they're off! The horses race toward the first turn at incredible speed.|An energetic male sports commentator, fast-paced and excited.
+vd009|||Once upon a time, in a land far away, lived a very clever fox.|A light, playful voice with childlike enthusiasm, bright and clear.
+vd010|||The ancient manuscript reveals secrets hidden for a thousand years.|A wise, measured elderly male voice, slow and deliberate, British English accent.
+vd011|||Good evening, ladies and gentlemen, and welcome to our show.|A sophisticated female voice with a slight French accent speaking English, elegant and refined.
+vd012|||System initialized. Running diagnostics. All systems nominal.|A clear, precise robotic-sounding voice, neutral and monotone with slight synthetic quality.
+vd013|||I hear what you are saying, and it is completely understandable to feel that way.|A warm, empathetic female therapist voice, calm and reassuring, unhurried pace.
+vd014|||Attention all units: proceed to grid reference seven-seven-alpha.|A firm, authoritative military male voice, clipped and commanding.
+vd015|||Oh my goodness, you have to try this amazing new recipe I just found!|An enthusiastic, bubbly female voice, high energy and friendly.
+vd016|||Dude, the waves were totally amazing out there today. Super happy about it!|A relaxed male voice with a California accent, casual and laid-back.
+vd017|||The quarterly results exceed expectations across all major metrics.|A sharp, businesslike female voice, confident and efficient, fast-paced delivery.
+vd018|||Chapter one. The morning sun filtered gently through the forest canopy.|A smooth, rich male audiobook narrator voice, expressive and engaging.
+vd019|||To be or not to be, that is the question.|A theatrical female voice, dramatic and expressive, stage projection quality.
+vd020|||And that is all for tonight. Stay well out there, everyone.|A warm, velvety male late-night radio DJ voice, smooth and intimate.

benchmarks/build_dataset/seed_tts_smoke/en/meta.lst

@@ -0,0 +1,20 @@
+smoke001|||The quick brown fox jumps over the lazy dog near the riverbank at sunset.
+smoke002|||Welcome to the future of text-to-speech synthesis in production systems.
+smoke003|||Yesterday the team finished rolling out the new authentication flow.
+smoke004|||She walked carefully across the wet cobblestones, careful not to slip.
+smoke005|||The conference call is scheduled for nine in the morning, Pacific time.
+smoke006|||Please remember to save your work before closing the editor.
+smoke007|||Two plus two equals four, but five hundred and forty three digits is long.
+smoke008|||I would like a coffee with oat milk and a chocolate croissant please.
+smoke009|||The library closes at eight on weekdays and six on Saturdays.
+smoke010|||During the Renaissance, art and science flourished in European cities.
+smoke011|||He whispered the secret word so quietly that no one else could hear.
+smoke012|||Our flight departs from gate twenty three at eleven fifteen.
+smoke013|||The storm knocked out power for six hours, but the backup generator kicked in.
+smoke014|||Reading a good book on a rainy afternoon is one of life's great pleasures.
+smoke015|||When the kettle whistled, she poured the hot water over the fresh tea leaves.
+smoke016|||The algorithm runs in linear time, which is a big improvement over the previous approach.
+smoke017|||In the distance, the mountains were shrouded in thick morning fog.
+smoke018|||Our company reported record revenue for the fourth quarter of the fiscal year.
+smoke019|||She explained the new policy in detail during the staff meeting this morning.
+smoke020|||The children laughed and played in the garden until the sun began to set.

benchmarks/glm_image/README.md

@@ -0,0 +1,157 @@
+# GLM-Image Benchmarks
+
+Benchmark GLM-Image T2I (text-to-image) and I2I (image-to-image) performance across three backends: HuggingFace baseline, vLLM-Omni offline, and vLLM-Omni online serving.
+
+## Benchmarks
+
+| Benchmark | Script | Description |
+|-----------|--------|-------------|
+| HuggingFace Baseline | `huggingface/inference.py` | Single-GPU transformers + diffusers pipeline |
+| vLLM-Omni Offline | `vllm-omni/inference.py` | Offline inference with continuous batching |
+| vLLM-Omni Online | `benchmark_glm_image.py` | Online serving via `/v1/chat/completions` |
+
+## HuggingFace Baseline
+
+Single-request sequential inference using the reference HuggingFace pipeline.
+
+```bash
+# T2I
+CUDA_VISIBLE_DEVICES=0 python benchmarks/glm_image/huggingface/inference.py \
+    --model-path /path/to/GLM-Image --mode t2i --num-prompts 10
+
+# I2I
+CUDA_VISIBLE_DEVICES=0 python benchmarks/glm_image/huggingface/inference.py \
+    --model-path /path/to/GLM-Image --mode i2i --num-prompts 10
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--model-path` | `zai-org/GLM-Image` | Model path |
+| `--mode` | `t2i` | `t2i` or `i2i` |
+| `--dataset-path` | `prompt/prompt.json` | Path to prompt.json |
+| `--num-prompts` | `10` | Number of images to generate |
+| `--width` / `--height` | `1024` | Output image size |
+| `--num-inference-steps` | `50` | Diffusion denoising steps |
+| `--output-dir` | `benchmarks/glm_image/huggingface/outputs` | Output directory |
+| `--output-file` | - | JSON file for metrics |
+
+## vLLM-Omni Offline
+
+Multi-GPU offline inference with pipeline parallelism and continuous batching.
+
+```bash
+# T2I
+CUDA_VISIBLE_DEVICES=0,1 python benchmarks/glm_image/vllm-omni/inference.py \
+    --model-path /path/to/GLM-Image --mode t2i --num-prompts 10
+
+# I2I
+CUDA_VISIBLE_DEVICES=0,1 python benchmarks/glm_image/vllm-omni/inference.py \
+    --model-path /path/to/GLM-Image --mode i2i --num-prompts 10
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--model-path` | `zai-org/GLM-Image` | Model path |
+| `--deploy-config` | - | Deploy config YAML |
+| `--mode` | `t2i` | `t2i` or `i2i` |
+| `--dataset-path` | `prompt/prompt.json` | Path to prompt.json |
+| `--num-prompts` | `10` | Number of images to generate |
+| `--width` / `--height` | `1024` | Output image size |
+| `--num-inference-steps` | `50` | Diffusion denoising steps |
+| `--output-dir` | `benchmarks/glm_image/vllm-omni/outputs` | Output directory |
+| `--output-file` | - | JSON file for metrics |
+| `--stage-init-timeout` | `600` | Stage initialization timeout (s) |
+
+### Latency Computation
+
+In offline mode all requests are submitted simultaneously and processed with continuous batching. The per-request latency is computed by summing the actual per-stage times (with `stage_0_gen_ms` diffed against the previous request to remove accumulated queue/scheduling wait).
+
+## vLLM-Omni Online Serving
+
+### Start the server
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 vllm serve /path/to/GLM-Image \
+    --omni --port 8091 --host 0.0.0.0 \
+    --served-model-name glm-image
+```
+
+### Run the benchmark
+
+```bash
+# T2I
+python benchmarks/glm_image/benchmark_glm_image.py \
+    --mode t2i --num-prompts 10 --model glm-image
+
+# I2I
+python benchmarks/glm_image/benchmark_glm_image.py \
+    --mode i2i --num-prompts 10 --model glm-image
+
+# Custom dataset
+python benchmarks/glm_image/benchmark_glm_image.py \
+    --mode i2i --dataset custom \
+    --dataset-path prompts.json --num-prompts 5
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--mode` | `t2i` | `t2i` or `i2i` |
+| `--dataset` | `prompt` | `prompt`, `random`, or `custom` |
+| `--dataset-path` | - | JSON file path (required for `custom`) |
+| `--num-prompts` | `10` | Number of benchmark requests |
+| `--max-concurrency` | `1` | Max concurrent requests |
+| `--request-rate` | `inf` | Requests per second (Poisson arrival) |
+| `--warmup-requests` | `1` | Warmup requests before measurement |
+| `--width` / `--height` | `1024` | Output image size |
+| `--num-inference-steps` | `50` | Diffusion denoising steps |
+| `--seed` | - | Random seed |
+| `--model` | `default` | Model name (must match `--served-model-name`) |
+| `--host` | `localhost` | Server host |
+| `--port` | `8091` | Server port |
+| `--output-file` | - | JSON output file for metrics |
+| `--num-input-images` | `1` | Number of input images for random I2I |
+
+## Dataset
+
+The default dataset is hosted on [HuggingFace](https://huggingface.co/datasets/JaredforReal/glm-image-bench) (`prompt.json`). It is automatically downloaded and cached to `prompt/prompt.json` on first run. No manual setup needed.
+
+Each entry contains:
+
+- `t2i_prompt`: Text prompt for text-to-image generation
+- `i2i_prompt`: Text prompt for image-to-image editing
+- `image_url`: Source image URL for I2I (downloaded and cached on first use)
+
+Custom datasets use the same JSON format and can be provided via `--dataset-path`.
+
+## Pipeline Timings
+
+All three benchmarks report per-stage pipeline timings (in milliseconds):
+
+| Key | Description |
+|-----|-------------|
+| `preprocess_ms` | Input preprocessing (tokenization, multimodal encoding) |
+| `stage_0_gen_ms` | AR (autoregressive) model generation time |
+| `ar2diffusion_ms` | AR output to diffusion input conversion |
+| `stage_1_gen_ms` | Diffusion model denoising time |
+| `queue_wait_ms` | Queue wait time before processing |
+
+The stages are ordered by execution: `preprocess → stage_0 (AR) → ar2diffusion → stage_1 (Diffusion)`.
+
+## Sample Results
+
+Tested on 2x GPU with 10 prompts, 1024x1024, 50 denoising steps:
+
+| Backend | Mode | Latency Mean (s) | Throughput (img/s) |
+|---------|------|-------------------|--------------------|
+| HuggingFace | T2I | 72.6 | 0.014 |
+| HuggingFace | I2I | 70.9 | 0.014 |
+| vLLM-Omni Offline | T2I | 35.0 | 0.044 |
+| vLLM-Omni Offline | I2I | 31.0 | 0.053 |
+| vLLM-Omni Online | T2I | 38.8 | 0.026 |
+| vLLM-Omni Online | I2I | 34.7 | 0.029 |

benchmarks/glm_image/benchmark_glm_image.py

@@ -0,0 +1,464 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Online serving benchmark for GLM-Image (T2I and I2I modes).
+
+Sends requests to the /v1/chat/completions endpoint and reports end-to-end
+latency, throughput, and per-stage durations (when the server is started with
+--enable-diffusion-pipeline-profiler and/or --enable-ar-profiler).
+
+Supports three dataset types:
+  - prompt:   Use prompt.json (default). T2I uses t2i_prompt, I2I uses i2i_prompt
+              and sends source images from image_url.
+  - random:   Generate synthetic prompts (and random images for I2I).
+  - custom:   Load from a user-specified JSON file.
+
+Usage:
+    # T2I with prompt.json (default)
+    python benchmarks/glm_image/benchmark_glm_image.py \
+        --mode t2i --num-prompts 10
+
+    # I2I with prompt.json (downloads source images automatically)
+    python benchmarks/glm_image/benchmark_glm_image.py \
+        --mode i2i --num-prompts 10
+
+    # Random dataset
+    python benchmarks/glm_image/benchmark_glm_image.py \
+        --mode t2i --dataset random --num-prompts 20
+
+    # Custom dataset
+    python benchmarks/glm_image/benchmark_glm_image.py \
+        --mode i2i --dataset custom \
+        --dataset-path my_prompts.json --num-prompts 5
+"""
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+import numpy as np
+import requests as sync_requests
+from PIL import Image
+from tqdm.asyncio import tqdm
+
+# Import backends from the diffusion benchmark (add parent dirs to path)
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "diffusion"))
+from backends import RequestFuncOutput
+
+BENCHMARK_DIR = Path(__file__).resolve().parent
+DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
+IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
+
+DATASET_REPO = "JaredforReal/glm-image-bench"
+DATASET_FILE = "prompt.json"
+
+
+def _ensure_prompt_json(dataset_path: str | None) -> str:
+    """Return path to prompt.json, downloading from HuggingFace if needed."""
+    if dataset_path:
+        return dataset_path
+    local = DEFAULT_PROMPT_JSON
+    if local.exists():
+        return str(local)
+    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
+    try:
+        from huggingface_hub import hf_hub_download
+
+        downloaded = hf_hub_download(
+            repo_id=DATASET_REPO,
+            filename=DATASET_FILE,
+            repo_type="dataset",
+        )
+        local.parent.mkdir(parents=True, exist_ok=True)
+        import shutil
+
+        shutil.copy2(downloaded, local)
+        print(f"Saved to {local}")
+    except ImportError:
+        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
+        import urllib.request
+
+        local.parent.mkdir(parents=True, exist_ok=True)
+        urllib.request.urlretrieve(url, local)
+        print(f"Saved to {local}")
+    return str(local)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class GLMImageRequest:
+    prompt: str
+    image_path: str | None = None  # Only for I2I mode
+
+
+def download_image(url: str) -> str:
+    """Download an image to cache and return the local path."""
+    IMAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    fname = url.rsplit("/", 1)[-1]
+    local_path = IMAGE_CACHE_DIR / fname
+    if local_path.exists():
+        return str(local_path)
+    resp = sync_requests.get(url, timeout=30)
+    resp.raise_for_status()
+    local_path.write_bytes(resp.content)
+    return str(local_path)
+
+
+def encode_image_as_data_url(path: str) -> str:
+    """Encode a local image file as a base64 data URL."""
+    with open(path, "rb") as f:
+        encoded = base64.b64encode(f.read()).decode("utf-8")
+    ext = Path(path).suffix.lower()
+    mime = {"png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}.get(ext, "image/png")
+    return f"data:{mime};base64,{encoded}"
+
+
+# ---------------------------------------------------------------------------
+# Datasets
+# ---------------------------------------------------------------------------
+
+
+class PromptDataset:
+    """Load from prompt.json. T2I uses t2i_prompt, I2I uses i2i_prompt + image_url."""
+
+    def __init__(self, args: argparse.Namespace):
+        path = _ensure_prompt_json(args.dataset_path)
+        with open(path, encoding="utf-8") as f:
+            raw = json.load(f)
+
+        prompt_key = "t2i_prompt" if args.mode == "t2i" else "i2i_prompt"
+        self.items: list[GLMImageRequest] = []
+
+        for entry in raw:
+            prompt = entry.get(prompt_key, "").strip()
+            if not prompt:
+                continue
+            image_path = None
+            if args.mode == "i2i":
+                url = entry.get("image_url", "")
+                if url:
+                    image_path = download_image(url)
+            self.items.append(GLMImageRequest(prompt=prompt, image_path=image_path))
+
+        if args.num_prompts and len(self.items) > args.num_prompts:
+            self.items = self.items[: args.num_prompts]
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def __getitem__(self, idx: int) -> GLMImageRequest:
+        return self.items[idx]
+
+    def get_requests(self) -> list[GLMImageRequest]:
+        return list(self.items)
+
+
+class RandomDataset:
+    """Generate synthetic prompts (and optional random images for I2I)."""
+
+    def __init__(self, args: argparse.Namespace):
+        self.args = args
+        self.num_prompts = args.num_prompts
+        self._random_image_paths: list[str] | None = None
+        if args.mode == "i2i":
+            self._random_image_paths = self._generate_random_images()
+
+    def _generate_random_images(self) -> list[str]:
+        paths: list[str] = []
+        for i in range(self.args.num_input_images):
+            img = Image.new("RGB", (512, 512), (128 + i * 30 % 128, 64, 192))
+            path = os.path.join(tempfile.gettempdir(), f"glm_image_bench_input_{i}.png")
+            img.save(path)
+            paths.append(path)
+        return paths
+
+    def __len__(self) -> int:
+        return self.num_prompts
+
+    def __getitem__(self, idx: int) -> GLMImageRequest:
+        image_path = None
+        if self._random_image_paths is not None:
+            image_path = self._random_image_paths[idx % len(self._random_image_paths)]
+        return GLMImageRequest(
+            prompt=f"A beautiful scene with vivid colors and intricate details, prompt {idx}",
+            image_path=image_path,
+        )
+
+    def get_requests(self) -> list[GLMImageRequest]:
+        return [self[i] for i in range(len(self))]
+
+
+class CustomDataset:
+    """Load from a user-specified JSON file.
+
+    Expected format:
+    [
+        {"prompt": "A cat sitting on a windowsill"},
+        {"prompt": "Make it look like winter", "image_path": "/path/to/img.png"}
+    ]
+    """
+
+    def __init__(self, args: argparse.Namespace):
+        if not args.dataset_path:
+            raise ValueError("--dataset-path is required for custom dataset")
+        with open(args.dataset_path, encoding="utf-8") as f:
+            raw = json.load(f)
+        self.items: list[GLMImageRequest] = []
+        for item in raw:
+            self.items.append(
+                GLMImageRequest(
+                    prompt=item.get("prompt", ""),
+                    image_path=item.get("image_path"),
+                )
+            )
+        if args.num_prompts and len(self.items) > args.num_prompts:
+            self.items = self.items[: args.num_prompts]
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def __getitem__(self, idx: int) -> GLMImageRequest:
+        return self.items[idx]
+
+    def get_requests(self) -> list[GLMImageRequest]:
+        return list(self.items)
+
+
+# ---------------------------------------------------------------------------
+# Async request for GLM-Image (chat completions with image support)
+# ---------------------------------------------------------------------------
+
+
+async def async_glm_image_request(
+    req: GLMImageRequest,
+    api_url: str,
+    model: str,
+    session: aiohttp.ClientSession,
+    pbar: Any,
+    args: argparse.Namespace,
+) -> RequestFuncOutput:
+    """Send a single T2I or I2I request via chat completions endpoint."""
+    output = RequestFuncOutput()
+    output.start_time = time.perf_counter()
+
+    # Build messages
+    if req.image_path and args.mode == "i2i":
+        data_url = encode_image_as_data_url(req.image_path)
+        content = [
+            {"type": "text", "text": req.prompt},
+            {"type": "image_url", "image_url": {"url": data_url}},
+        ]
+    else:
+        content = req.prompt
+
+    messages = [{"role": "user", "content": content}]
+
+    extra_body: dict[str, Any] = {}
+    if args.height:
+        extra_body["height"] = args.height
+    if args.width:
+        extra_body["width"] = args.width
+    if args.num_inference_steps:
+        extra_body["num_inference_steps"] = args.num_inference_steps
+    if args.seed is not None:
+        extra_body["seed"] = args.seed
+
+    payload: dict[str, Any] = {
+        "model": model,
+        "messages": messages,
+    }
+    if extra_body:
+        payload["extra_body"] = extra_body
+
+    try:
+        async with session.post(api_url, json=payload) as response:
+            if response.status == 200:
+                resp_json = await response.json()
+                output.response_body = resp_json
+                output.success = True
+                try:
+                    choices = resp_json.get("choices", [])
+                    if choices and isinstance(choices, list):
+                        msg = choices[0].get("message", {})
+                        if isinstance(msg, dict):
+                            resp_content = msg.get("content", [])
+                            if resp_content and isinstance(resp_content, list) and len(resp_content) > 0:
+                                first_item = resp_content[0]
+                                if isinstance(first_item, dict):
+                                    output.stage_durations = first_item.get("stage_durations") or {}
+                                    output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
+                except (IndexError, TypeError, AttributeError):
+                    pass
+            else:
+                output.error = f"HTTP {response.status}: {await response.text()}"
+                output.success = False
+    except Exception as e:
+        output.error = str(e)
+        output.success = False
+
+    output.latency = time.perf_counter() - output.start_time
+    if pbar:
+        pbar.update(1)
+    return output
+
+
+# ---------------------------------------------------------------------------
+# Benchmark
+# ---------------------------------------------------------------------------
+
+
+async def iter_requests(n: int, request_rate: float) -> Any:
+    import random as _random
+
+    for i in range(n):
+        if request_rate != float("inf") and i > 0:
+            await asyncio.sleep(_random.expovariate(request_rate))
+        yield i
+
+
+def calculate_metrics(outputs: list[RequestFuncOutput], total_duration: float) -> dict[str, Any]:
+    success = [o for o in outputs if o.success]
+    errors = [o for o in outputs if not o.success]
+    latencies = [o.latency for o in success]
+    peak_mems = [o.peak_memory_mb for o in success if o.peak_memory_mb > 0]
+
+    stage_duration_lists: dict[str, list[float]] = {}
+    for o in success:
+        for stage, dur in (o.stage_durations or {}).items():
+            stage_duration_lists.setdefault(stage, []).append(dur)
+
+    return {
+        "duration": total_duration,
+        "completed_requests": len(success),
+        "failed_requests": len(errors),
+        "throughput_qps": len(success) / total_duration if total_duration > 0 else 0,
+        "latency_mean": float(np.mean(latencies)) if latencies else 0,
+        "latency_median": float(np.median(latencies)) if latencies else 0,
+        "latency_p99": float(np.percentile(latencies, 99)) if latencies else 0,
+        "latency_p95": float(np.percentile(latencies, 95)) if latencies else 0,
+        "peak_memory_mb_max": max(peak_mems) if peak_mems else 0,
+        "stage_durations_mean": {s: float(np.mean(v)) for s, v in stage_duration_lists.items()},
+        "stage_durations_p50": {s: float(np.percentile(v, 50)) for s, v in stage_duration_lists.items()},
+    }
+
+
+async def benchmark(args: argparse.Namespace) -> None:
+    api_url = f"http://{args.host}:{args.port}/v1/chat/completions"
+
+    # Load dataset
+    if args.dataset == "prompt":
+        dataset = PromptDataset(args)
+    elif args.dataset == "random":
+        dataset = RandomDataset(args)
+    elif args.dataset == "custom":
+        dataset = CustomDataset(args)
+    else:
+        raise ValueError(f"Unknown dataset: {args.dataset}")
+
+    glm_requests = dataset.get_requests()
+    print(f"Prepared {len(glm_requests)} requests (mode={args.mode}, dataset={args.dataset})")
+
+    semaphore = asyncio.Semaphore(args.max_concurrency) if args.max_concurrency else None
+
+    async def limited_request(idx: int, req: GLMImageRequest, session: aiohttp.ClientSession, pbar: Any):
+        if semaphore:
+            async with semaphore:
+                return await async_glm_image_request(req, api_url, args.model, session, pbar, args)
+        return await async_glm_image_request(req, api_url, args.model, session, pbar, args)
+
+    async with aiohttp.ClientSession() as session:
+        # Warmup
+        if args.warmup_requests and glm_requests:
+            print(f"Running {args.warmup_requests} warmup request(s)...")
+            for i in range(args.warmup_requests):
+                await limited_request(i, glm_requests[i % len(glm_requests)], session, None)
+
+        # Main benchmark
+        pbar = tqdm(total=len(glm_requests), disable=args.disable_tqdm)
+        start_time = time.perf_counter()
+        tasks = []
+        async for idx in iter_requests(len(glm_requests), args.request_rate):
+            tasks.append(asyncio.create_task(limited_request(idx, glm_requests[idx], session, pbar)))
+        outputs = await asyncio.gather(*tasks)
+        total_duration = time.perf_counter() - start_time
+        pbar.close()
+
+    # Metrics
+    metrics = calculate_metrics(outputs, total_duration)
+    metrics["mode"] = args.mode
+    metrics["model"] = args.model
+    metrics["dataset"] = args.dataset
+
+    print(f"\n{' GLM-Image Online Benchmark Result ':=^60}")
+    print(f"{'Mode:':<40} {args.mode}")
+    print(f"{'Model:':<40} {args.model}")
+    print(f"{'Dataset:':<40} {args.dataset}")
+    print("-" * 50)
+    print(f"{'Benchmark duration (s):':<40} {metrics['duration']:.2f}")
+    print(f"{'Request rate:':<40} {args.request_rate}")
+    print(f"{'Max concurrency:':<40} {args.max_concurrency}")
+    print(f"{'Successful requests:':<40} {metrics['completed_requests']}/{len(glm_requests)}")
+    print("-" * 50)
+    print(f"{'Throughput (req/s):':<40} {metrics['throughput_qps']:.2f}")
+    print(f"{'Latency Mean (s):':<40} {metrics['latency_mean']:.4f}")
+    print(f"{'Latency Median (s):':<40} {metrics['latency_median']:.4f}")
+    print(f"{'Latency P95 (s):':<40} {metrics['latency_p95']:.4f}")
+    print(f"{'Latency P99 (s):':<40} {metrics['latency_p99']:.4f}")
+
+    if metrics["peak_memory_mb_max"] > 0:
+        print("-" * 50)
+        print(f"{'Peak Memory Max (MB):':<40} {metrics['peak_memory_mb_max']:.2f}")
+
+    if metrics["stage_durations_mean"]:
+        print("-" * 50)
+        print("Stage Durations Mean:")
+        for stage, val in sorted(metrics["stage_durations_mean"].items()):
+            unit = "ms" if stage.endswith("_ms") else "s"
+            print(f"  {stage + ':':<38} {val:.4f} ({unit})")
+
+    print("=" * 60)
+
+    if args.output_file:
+        with open(args.output_file, "w") as f:
+            json.dump(metrics, f, indent=2)
+        print(f"Metrics saved to {args.output_file}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Benchmark GLM-Image T2I/I2I online serving.")
+    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
+    parser.add_argument("--dataset", type=str, default="prompt", choices=["prompt", "random", "custom"])
+    parser.add_argument("--dataset-path", type=str, default=None)
+    parser.add_argument("--num-prompts", type=int, default=10)
+    parser.add_argument("--max-concurrency", type=int, default=1)
+    parser.add_argument("--request-rate", type=float, default=float("inf"))
+    parser.add_argument("--warmup-requests", type=int, default=1)
+    parser.add_argument("--width", type=int, default=1024)
+    parser.add_argument("--height", type=int, default=1024)
+    parser.add_argument("--num-inference-steps", type=int, default=50)
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--model", type=str, default="default")
+    parser.add_argument("--host", type=str, default="localhost")
+    parser.add_argument("--port", type=int, default=8091)
+    parser.add_argument("--output-file", type=str, default=None)
+    parser.add_argument("--disable-tqdm", action="store_true")
+    parser.add_argument("--num-input-images", type=int, default=1, help="For random I2I dataset.")
+    args = parser.parse_args()
+    asyncio.run(benchmark(args))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/glm_image/huggingface/inference.py

@@ -0,0 +1,291 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+HuggingFace (transformers + diffusers) baseline benchmark for GLM-Image.
+
+Supports T2I and I2I modes with the prompt.json dataset.
+Downloads source images for I2I from image_url on first run and caches locally.
+
+Usage:
+    # T2I mode (text-to-image, no source images needed)
+    python benchmarks/glm_image/huggingface/inference.py \
+        --model-path zai-org/GLM-Image \
+        --mode t2i --num-prompts 10
+
+    # I2I mode (image-to-image, downloads source images)
+    python benchmarks/glm_image/huggingface/inference.py \
+        --model-path zai-org/GLM-Image \
+        --mode i2i --num-prompts 10
+
+    # With custom prompt.json
+    python benchmarks/glm_image/huggingface/inference.py \
+        --model-path zai-org/GLM-Image \
+        --mode i2i --dataset-path prompts.json --num-prompts 5
+"""
+
+import argparse
+import json
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import requests
+import torch
+from PIL import Image
+
+BENCHMARK_DIR = Path(__file__).resolve().parent.parent
+DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
+IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
+
+DATASET_REPO = "JaredforReal/glm-image-bench"
+DATASET_FILE = "prompt.json"
+
+
+def _ensure_prompt_json(dataset_path: str | None) -> str:
+    """Return path to prompt.json, downloading from HuggingFace if needed."""
+    if dataset_path:
+        return dataset_path
+    local = DEFAULT_PROMPT_JSON
+    if local.exists():
+        return str(local)
+    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
+    try:
+        from huggingface_hub import hf_hub_download
+
+        downloaded = hf_hub_download(
+            repo_id=DATASET_REPO,
+            filename=DATASET_FILE,
+            repo_type="dataset",
+        )
+        local.parent.mkdir(parents=True, exist_ok=True)
+        import shutil
+
+        shutil.copy2(downloaded, local)
+        print(f"Saved to {local}")
+    except ImportError:
+        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
+        import urllib.request
+
+        local.parent.mkdir(parents=True, exist_ok=True)
+        urllib.request.urlretrieve(url, local)
+        print(f"Saved to {local}")
+    return str(local)
+
+
+HEIGHT = 1024
+WIDTH = 1024
+SEED = 42
+NUM_INFERENCE_STEPS = 50
+GUIDANCE_SCALE = 1.5
+
+
+# ---------------------------------------------------------------------------
+# Dataset
+# ---------------------------------------------------------------------------
+
+
+def load_dataset(
+    dataset_path: str | None,
+    mode: str,
+    num_prompts: int,
+) -> list[dict]:
+    """Load prompts from prompt.json and prepare per-request data."""
+    path = _ensure_prompt_json(dataset_path)
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+
+    items = []
+    for entry in raw:
+        if mode == "t2i":
+            prompt_key = "t2i_prompt"
+        else:
+            prompt_key = "i2i_prompt"
+
+        prompt_text = entry.get(prompt_key, "").strip()
+        if not prompt_text:
+            continue
+
+        item = {"prompt": prompt_text}
+        if mode == "i2i":
+            item["image_url"] = entry.get("image_url", "")
+        items.append(item)
+
+    if num_prompts and len(items) > num_prompts:
+        items = items[:num_prompts]
+    return items
+
+
+def download_image(url: str, cache_dir: Path) -> str:
+    """Download an image to cache_dir and return the local path."""
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    fname = url.rsplit("/", 1)[-1]
+    local_path = cache_dir / fname
+    if local_path.exists():
+        return str(local_path)
+    print(f"  Downloading {url} ...")
+    resp = requests.get(url, timeout=30)
+    resp.raise_for_status()
+    local_path.write_bytes(resp.content)
+    return str(local_path)
+
+
+# ---------------------------------------------------------------------------
+# Benchmark
+# ---------------------------------------------------------------------------
+
+
+def benchmark(args: argparse.Namespace) -> None:
+    from diffusers.pipelines.glm_image import GlmImagePipeline
+
+    print("=" * 60)
+    print("GLM-Image HuggingFace Baseline Benchmark")
+    print(f"Mode: {args.mode}  |  Model: {args.model_path}")
+    print(f"Size: {args.height}x{args.width}  |  Steps: {args.num_inference_steps}")
+    print("=" * 60)
+
+    # Load dataset
+    items = load_dataset(args.dataset_path, args.mode, args.num_prompts)
+    if not items:
+        print("No prompts loaded. Exiting.")
+        return
+    print(f"Loaded {len(items)} prompts for {args.mode} mode")
+
+    # Download I2I source images
+    if args.mode == "i2i":
+        print("Preparing source images...")
+        for item in items:
+            url = item.get("image_url", "")
+            if url:
+                item["image_path"] = download_image(url, IMAGE_CACHE_DIR)
+            else:
+                item["image_path"] = None
+
+    # Load pipeline
+    print(f"\nLoading pipeline from {args.model_path} ...")
+    t0 = time.perf_counter()
+    pipe = GlmImagePipeline.from_pretrained(
+        args.model_path,
+        torch_dtype=torch.bfloat16,
+        device_map="cuda",
+    )
+    init_time = time.perf_counter() - t0
+    print(f"Pipeline loaded in {init_time:.2f}s")
+
+    # Create output dir
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Run benchmark
+    generator = torch.Generator(device="cuda").manual_seed(args.seed)
+    latencies = []
+    success = 0
+    failed = 0
+
+    print(f"\nRunning {len(items)} requests sequentially...")
+    print("-" * 60)
+
+    for i, item in enumerate(items):
+        prompt = item["prompt"]
+        gen_kwargs: dict = {
+            "prompt": prompt,
+            "height": args.height,
+            "width": args.width,
+            "num_inference_steps": args.num_inference_steps,
+            "guidance_scale": args.guidance_scale,
+            "generator": generator,
+        }
+
+        if args.mode == "i2i":
+            img_path = item.get("image_path")
+            if img_path and os.path.exists(img_path):
+                gen_kwargs["image"] = [Image.open(img_path).convert("RGB")]
+            else:
+                print(f"  [{i + 1}] SKIP: no source image")
+                failed += 1
+                continue
+
+        t_start = time.perf_counter()
+        try:
+            result = pipe(**gen_kwargs)
+            image = result.images[0]
+            elapsed = time.perf_counter() - t_start
+            latencies.append(elapsed)
+            success += 1
+
+            out_path = os.path.join(args.output_dir, f"{i:04d}.png")
+            image.save(out_path)
+            print(f"  [{i + 1}/{len(items)}] {elapsed:.3f}s -> {out_path}")
+        except Exception as e:
+            elapsed = time.perf_counter() - t_start
+            failed += 1
+            print(f"  [{i + 1}/{len(items)}] FAILED ({elapsed:.3f}s): {e}")
+
+    # Report
+    total_gen_time = sum(latencies) if latencies else 0
+    print("\n" + "=" * 60)
+    print("HuggingFace Baseline Results")
+    print("=" * 60)
+    print(f"{'Mode:':<40} {args.mode}")
+    print(f"{'Model:':<40} {args.model_path}")
+    print(f"{'Image size:':<40} {args.height}x{args.width}")
+    print(f"{'Num inference steps:':<40} {args.num_inference_steps}")
+    print("-" * 50)
+    print(f"{'Pipeline init time (s):':<40} {init_time:.2f}")
+    print(f"{'Successful:':<40} {success}/{len(items)}")
+    print(f"{'Failed:':<40} {failed}")
+    print("-" * 50)
+    if latencies:
+        arr = np.array(latencies)
+        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
+        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
+        print(f"{'Latency Mean (s):':<40} {arr.mean():.4f}")
+        print(f"{'Latency Median (s):':<40} {np.median(arr):.4f}")
+        print(f"{'Latency P95 (s):':<40} {np.percentile(arr, 95):.4f}")
+        print(f"{'Latency P99 (s):':<40} {np.percentile(arr, 99):.4f}")
+
+    print(f"\n{'Output dir:':<40} {args.output_dir}")
+    print("=" * 60)
+
+    # Save metrics JSON
+    metrics = {
+        "backend": "huggingface",
+        "mode": args.mode,
+        "model": args.model_path,
+        "height": args.height,
+        "width": args.width,
+        "num_inference_steps": args.num_inference_steps,
+        "init_time_s": init_time,
+        "completed_requests": success,
+        "failed_requests": failed,
+        "total_gen_time_s": total_gen_time,
+        "throughput_qps": success / total_gen_time if total_gen_time > 0 else 0,
+        "latency_mean": float(np.mean(latencies)) if latencies else 0,
+        "latency_median": float(np.median(latencies)) if latencies else 0,
+        "latency_p95": float(np.percentile(latencies, 95)) if latencies else 0,
+        "latency_p99": float(np.percentile(latencies, 99)) if latencies else 0,
+    }
+    if args.output_file:
+        with open(args.output_file, "w") as f:
+            json.dump(metrics, f, indent=2)
+        print(f"Metrics saved to {args.output_file}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="GLM-Image HuggingFace baseline benchmark")
+    parser.add_argument("--model-path", type=str, default="zai-org/GLM-Image")
+    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
+    parser.add_argument("--dataset-path", type=str, default=None, help="Path to prompt.json")
+    parser.add_argument("--num-prompts", type=int, default=10)
+    parser.add_argument("--height", type=int, default=HEIGHT)
+    parser.add_argument("--width", type=int, default=WIDTH)
+    parser.add_argument("--num-inference-steps", type=int, default=NUM_INFERENCE_STEPS)
+    parser.add_argument("--guidance-scale", type=float, default=GUIDANCE_SCALE)
+    parser.add_argument("--seed", type=int, default=SEED)
+    parser.add_argument("--output-dir", type=str, default="benchmarks/glm_image/huggingface/outputs")
+    parser.add_argument("--output-file", type=str, default=None, help="JSON file for metrics")
+    args = parser.parse_args()
+    benchmark(args)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/glm_image/vllm-omni/inference.py

@@ -0,0 +1,505 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+vLLM-Omni offline benchmark for GLM-Image.
+
+Supports T2I and I2I modes with the prompt.json dataset.
+Downloads source images for I2I from image_url on first run and caches locally.
+
+Usage:
+    # T2I mode
+    python benchmarks/glm_image/vllm-omni/inference.py \
+        --model-path zai-org/GLM-Image \
+        --mode t2i --num-prompts 10
+
+    # I2I mode (downloads source images)
+    python benchmarks/glm_image/vllm-omni/inference.py \
+        --model-path zai-org/GLM-Image \
+        --mode i2i --num-prompts 10
+"""
+
+import argparse
+import json
+import math
+import os
+import time
+from pathlib import Path
+
+import numpy as np
+import requests
+from PIL import Image
+from vllm import SamplingParams
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+BENCHMARK_DIR = Path(__file__).resolve().parent.parent
+DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
+IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
+DEFAULT_DEPLOY_CONFIG = "vllm_omni/deploy/glm_image.yaml"
+
+DATASET_REPO = "JaredforReal/glm-image-bench"
+DATASET_FILE = "prompt.json"
+
+
+def _ensure_prompt_json(dataset_path: str | None) -> str:
+    """Return path to prompt.json, downloading from HuggingFace if needed."""
+    if dataset_path:
+        return dataset_path
+    local = DEFAULT_PROMPT_JSON
+    if local.exists():
+        return str(local)
+    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
+    try:
+        from huggingface_hub import hf_hub_download
+
+        downloaded = hf_hub_download(
+            repo_id=DATASET_REPO,
+            filename=DATASET_FILE,
+            repo_type="dataset",
+        )
+        local.parent.mkdir(parents=True, exist_ok=True)
+        import shutil
+
+        shutil.copy2(downloaded, local)
+        print(f"Saved to {local}")
+    except ImportError:
+        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
+        import urllib.request
+
+        local.parent.mkdir(parents=True, exist_ok=True)
+        urllib.request.urlretrieve(url, local)
+        print(f"Saved to {local}")
+    return str(local)
+
+
+SEED = 42
+HEIGHT = 1024
+WIDTH = 1024
+NUM_INFERENCE_STEPS = 50
+GUIDANCE_SCALE = 1.5
+
+GLM_IMAGE_EOS_TOKEN_ID = 16385
+GLM_IMAGE_VISION_VOCAB_SIZE = 16512
+
+
+# ---------------------------------------------------------------------------
+# Dataset
+# ---------------------------------------------------------------------------
+
+
+def load_dataset(
+    dataset_path: str | None,
+    mode: str,
+    num_prompts: int,
+) -> list[dict]:
+    path = _ensure_prompt_json(dataset_path)
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+
+    items = []
+    for entry in raw:
+        prompt_key = "t2i_prompt" if mode == "t2i" else "i2i_prompt"
+        prompt_text = entry.get(prompt_key, "").strip()
+        if not prompt_text:
+            continue
+
+        item = {"prompt": prompt_text}
+        if mode == "i2i":
+            item["image_url"] = entry.get("image_url", "")
+        items.append(item)
+
+    if num_prompts and len(items) > num_prompts:
+        items = items[:num_prompts]
+    return items
+
+
+def download_image(url: str, cache_dir: Path) -> str:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    fname = url.rsplit("/", 1)[-1]
+    local_path = cache_dir / fname
+    if local_path.exists():
+        return str(local_path)
+    print(f"  Downloading {url} ...")
+    resp = requests.get(url, timeout=30)
+    resp.raise_for_status()
+    local_path.write_bytes(resp.content)
+    return str(local_path)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def compute_max_tokens(height: int, width: int, is_i2i: bool = False) -> int:
+    factor = 32
+    token_h = height // factor
+    token_w = width // factor
+    large_tokens = token_h * token_w
+
+    # Small preview tokens (half resolution in each dimension)
+
+    ratio = token_h / token_w if token_w > 0 else 1.0
+    small_token_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
+    small_token_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
+    small_tokens = small_token_h * small_token_w
+
+    # Mode-dependent totals:
+    # - t2i: small + large + EOS
+    # - i2i: large + EOS
+    if is_i2i:
+        return large_tokens + 1
+    return small_tokens + large_tokens + 1
+
+
+def build_prompt_t2i(prompt: str, height: int, width: int, **gen_kw) -> dict:
+    return {
+        "prompt": prompt,
+        "height": height,
+        "width": width,
+        "mm_processor_kwargs": {"target_h": height, "target_w": width},
+        **gen_kw,
+    }
+
+
+def build_prompt_i2i(prompt: str, image_path: str, height: int, width: int, **gen_kw) -> dict:
+    return {
+        "prompt": prompt,
+        "height": height,
+        "width": width,
+        "mm_processor_kwargs": {"target_h": height, "target_w": width},
+        "multi_modal_data": {"image": Image.open(image_path).convert("RGB")},
+        **gen_kw,
+    }
+
+
+def resolve_deploy_config(args: argparse.Namespace) -> str:
+    if args.deploy_config:
+        return args.deploy_config
+    if os.path.exists(DEFAULT_DEPLOY_CONFIG):
+        return DEFAULT_DEPLOY_CONFIG
+    fallback = Path(__file__).resolve().parents[3] / DEFAULT_DEPLOY_CONFIG
+    if fallback.exists():
+        return str(fallback)
+    raise FileNotFoundError("Deploy config not found. Specify --deploy-config.")
+
+
+# ---------------------------------------------------------------------------
+# Benchmark
+# ---------------------------------------------------------------------------
+
+
+def benchmark(args: argparse.Namespace) -> None:
+    is_i2i = args.mode == "i2i"
+
+    print("=" * 60)
+    print("GLM-Image vLLM-Omni Benchmark")
+    print(f"Mode: {args.mode}  |  Model: {args.model_path}")
+    print(f"Size: {args.height}x{args.width}  |  Steps: {args.num_inference_steps}")
+    print("=" * 60)
+
+    # Load dataset
+    items = load_dataset(args.dataset_path, args.mode, args.num_prompts)
+    if not items:
+        print("No prompts loaded. Exiting.")
+        return
+    print(f"Loaded {len(items)} prompts for {args.mode} mode")
+
+    # Download I2I source images
+    if is_i2i:
+        print("Preparing source images...")
+        for item in items:
+            url = item.get("image_url", "")
+            if url:
+                item["image_path"] = download_image(url, IMAGE_CACHE_DIR)
+            else:
+                item["image_path"] = None
+
+    # Init Omni
+    deploy_config = resolve_deploy_config(args)
+    print(f"\nInitializing vLLM-Omni (deploy config: {deploy_config}) ...")
+    t0 = time.perf_counter()
+
+    omni = Omni(
+        model=args.model_path,
+        deploy_config=deploy_config,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+        enable_ar_profiler=args.enable_ar_profiler,
+    )
+
+    init_time = time.perf_counter() - t0
+    print(f"Initialized in {init_time:.2f}s")
+
+    # Sampling params
+    max_tokens = compute_max_tokens(args.height, args.width, is_i2i=is_i2i)
+    ar_params = SamplingParams(
+        temperature=0.9,
+        top_p=0.75,
+        top_k=GLM_IMAGE_VISION_VOCAB_SIZE,
+        max_tokens=max_tokens,
+        stop_token_ids=[GLM_IMAGE_EOS_TOKEN_ID],
+        seed=args.seed,
+        detokenize=False,
+        extra_args={"target_h": args.height, "target_w": args.width},
+    )
+    diff_params = OmniDiffusionSamplingParams(
+        num_inference_steps=args.num_inference_steps,
+        guidance_scale=args.guidance_scale,
+        height=args.height,
+        width=args.width,
+        seed=args.seed,
+    )
+    sampling_params_list = [ar_params, diff_params]
+
+    # Build all prompts
+    gen_kw = {
+        "seed": args.seed,
+        "num_inference_steps": args.num_inference_steps,
+        "guidance_scale": args.guidance_scale,
+    }
+    all_prompts = []
+    for item in items:
+        if is_i2i:
+            img_path = item.get("image_path")
+            if not img_path or not os.path.exists(img_path):
+                continue
+            all_prompts.append(build_prompt_i2i(item["prompt"], img_path, args.height, args.width, **gen_kw))
+        else:
+            all_prompts.append(build_prompt_t2i(item["prompt"], args.height, args.width, **gen_kw))
+
+    valid = len(all_prompts)
+    print(f"Valid prompts: {valid}")
+
+    # Create output dir
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Warmup: run 1 request to prime caches, CUDA graphs, etc.
+    if all_prompts:
+        print("Running warmup request...")
+        try:
+            warmup_prompt = [all_prompts[0]]
+            omni.generate(warmup_prompt, sampling_params_list, py_generator=False)
+            print("Warmup done.\n")
+        except Exception as e:
+            print(f"Warmup failed (continuing): {e}")
+
+    # Run
+    print(f"\nRunning {valid} requests...")
+    print("-" * 60)
+
+    latencies = []
+    all_stage_durations: list[dict[str, float]] = []
+    success = 0
+    failed = 0
+    wall_start = time.perf_counter()
+
+    try:
+        output_idx = 0
+        for stage_outputs in omni.generate(all_prompts, sampling_params_list, py_generator=True):
+            if stage_outputs.final_output_type == "image":
+                request_output = stage_outputs.request_output
+                request_id = getattr(request_output, "request_id", "")
+
+                images = getattr(request_output, "images", [])
+                if not images and hasattr(request_output, "multimodal_output"):
+                    mm = request_output.multimodal_output
+                    if isinstance(mm, dict):
+                        images = mm.get("images", [])
+
+                elapsed = time.perf_counter() - wall_start
+                if images:
+                    for img in images:
+                        if isinstance(img, Image.Image):
+                            out_path = os.path.join(args.output_dir, f"{output_idx:04d}.png")
+                            img.save(out_path)
+                    success += 1
+                    latencies.append(elapsed)
+                    stage_durations = getattr(stage_outputs, "stage_durations", {})
+                    if stage_durations:
+                        all_stage_durations.append(stage_durations)
+                    # Show wall-clock elapsed and pipeline breakdown if available
+                    preprocess_str = ""
+                    if "preprocess_ms" in stage_durations:
+                        preprocess_str = f" preprocess={stage_durations['preprocess_ms'] / 1000.0:.2f}s"
+                    print(f"  [{success}/{valid}] id={request_id[:8]} {elapsed:.2f}s{preprocess_str}")
+                    output_idx += 1
+                else:
+                    failed += 1
+    except Exception as e:
+        print(f"Error: {e}")
+        failed = valid - success
+
+    total_gen_time = time.perf_counter() - wall_start
+
+    # Diff stage_0_gen_ms with previous request to remove accumulated wait time.
+    # stage_0_gen_ms is measured from submit_ts (same for all requests submitted
+    # at once), so it accumulates queue/scheduling overhead across requests.
+    # Other stages and pipeline timings are per-request already.
+    _TIMING_ORDER = [
+        "preprocess_ms",
+        "stage_0_gen_ms",
+        "ar2diffusion_ms",
+        "stage_1_gen_ms",
+        "queue_wait_ms",
+    ]
+
+    per_request_actual: list[dict[str, float]] = []
+    prev_stage_0_ms = 0.0
+    for sd in all_stage_durations:
+        actual = dict(sd)
+        s0 = sd.get("stage_0_gen_ms", 0.0)
+        actual["stage_0_gen_ms"] = s0 - prev_stage_0_ms
+        prev_stage_0_ms = s0
+        per_request_actual.append(actual)
+
+    per_request_e2e_ms: list[float] = []
+    for actual in per_request_actual:
+        e2e_ms = sum(v for k, v in actual.items() if k in _TIMING_ORDER)
+        if e2e_ms > 0:
+            per_request_e2e_ms.append(e2e_ms)
+
+    # Report
+    print("\n" + "=" * 60)
+    print("vLLM-Omni Benchmark Results")
+    print("=" * 60)
+    print(f"{'Mode:':<40} {args.mode}")
+    print(f"{'Model:':<40} {args.model_path}")
+    print(f"{'Image size:':<40} {args.height}x{args.width}")
+    print(f"{'Num inference steps:':<40} {args.num_inference_steps}")
+    print("-" * 50)
+    print(f"{'Init time (s):':<40} {init_time:.2f}")
+    print(f"{'Successful:':<40} {success}/{valid}")
+    print(f"{'Failed:':<40} {failed}")
+    print("-" * 50)
+
+    if per_request_e2e_ms:
+        per_request_s = np.array(per_request_e2e_ms) / 1000.0
+        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
+        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
+        print(f"{'Latency Mean (s):':<40} {per_request_s.mean():.4f}")
+        print(f"{'Latency Median (s):':<40} {np.median(per_request_s):.4f}")
+        print(f"{'Latency P95 (s):':<40} {np.percentile(per_request_s, 95):.4f}")
+        print(f"{'Latency P99 (s):':<40} {np.percentile(per_request_s, 99):.4f}")
+        print(f"{'Latency Min (s):':<40} {per_request_s.min():.4f}")
+        print(f"{'Latency Max (s):':<40} {per_request_s.max():.4f}")
+    elif latencies:
+        per_request = np.diff([0.0] + list(latencies))
+        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
+        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
+        print(f"{'Latency Mean (s) [wall-clock]:':<40} {per_request.mean():.4f}")
+        print(f"{'Latency Median (s) [wall-clock]:':<40} {np.median(per_request):.4f}")
+        print(f"{'Latency P95 (s) [wall-clock]:':<40} {np.percentile(per_request, 95):.4f}")
+        print(f"{'Latency P99 (s) [wall-clock]:':<40} {np.percentile(per_request, 99):.4f}")
+        print(f"{'Latency Min (s) [wall-clock]:':<40} {per_request.min():.4f}")
+        print(f"{'Latency Max (s) [wall-clock]:':<40} {per_request.max():.4f}")
+
+    if per_request_actual:
+        print("-" * 50)
+        print("Pipeline Timings Mean:")
+        for key in _TIMING_ORDER:
+            vals = [d.get(key, 0.0) for d in per_request_actual]
+            if any(v != 0 for v in vals):
+                unit = "ms" if key.endswith("_ms") else "s"
+                print(f"  {key + ':':<38} {np.mean(vals):.4f} ({unit})")
+        # Show any extra keys not in the ordered list
+        ordered_set = set(_TIMING_ORDER)
+        extra_keys = sorted(k for k in per_request_actual[0].keys() if k not in ordered_set)
+        for key in extra_keys:
+            vals = [d.get(key, 0.0) for d in per_request_actual]
+            if any(v != 0 for v in vals):
+                unit = "ms" if key.endswith("_ms") else "s"
+                print(f"  {key + ':':<38} {np.mean(vals):.4f} ({unit})")
+
+    print(f"\n{'Output dir:':<40} {args.output_dir}")
+    print("=" * 60)
+
+    # Metrics JSON
+    metrics = {
+        "backend": "vllm-omni",
+        "mode": args.mode,
+        "model": args.model_path,
+        "height": args.height,
+        "width": args.width,
+        "num_inference_steps": args.num_inference_steps,
+        "init_time_s": init_time,
+        "completed_requests": success,
+        "failed_requests": failed,
+        "total_gen_time_s": total_gen_time,
+        "throughput_qps": success / total_gen_time if total_gen_time > 0 else 0,
+    }
+    if per_request_e2e_ms:
+        per_request_s = np.array(per_request_e2e_ms) / 1000.0
+        metrics["latency_mean"] = float(per_request_s.mean())
+        metrics["latency_median"] = float(np.median(per_request_s))
+        metrics["latency_p95"] = float(np.percentile(per_request_s, 95))
+        metrics["latency_p99"] = float(np.percentile(per_request_s, 99))
+    elif latencies:
+        per_request = np.diff([0.0] + list(latencies))
+        metrics["latency_mean"] = float(per_request.mean())
+        metrics["latency_median"] = float(np.median(per_request))
+        metrics["latency_p95"] = float(np.percentile(per_request, 95))
+        metrics["latency_p99"] = float(np.percentile(per_request, 99))
+    else:
+        metrics["latency_mean"] = 0
+        metrics["latency_median"] = 0
+        metrics["latency_p95"] = 0
+        metrics["latency_p99"] = 0
+    if per_request_actual:
+        all_keys = list(_TIMING_ORDER) + sorted(k for k in per_request_actual[0].keys() if k not in set(_TIMING_ORDER))
+        stage_metrics = {}
+        for key in all_keys:
+            vals = [d.get(key, 0.0) for d in per_request_actual]
+            stage_metrics[key] = {
+                "mean": float(np.mean(vals)),
+                "median": float(np.median(vals)),
+                "p95": float(np.percentile(vals, 95)),
+            }
+        metrics["stage_durations"] = stage_metrics
+    if args.output_file:
+        with open(args.output_file, "w") as f:
+            json.dump(metrics, f, indent=2)
+        print(f"Metrics saved to {args.output_file}")
+
+    omni.close()
+    print("Done!")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="GLM-Image vLLM-Omni offline benchmark")
+    parser.add_argument("--model-path", type=str, default="zai-org/GLM-Image")
+    parser.add_argument("--deploy-config", type=str, default=None, help="Deploy config YAML")
+    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
+    parser.add_argument("--dataset-path", type=str, default=None, help="Path to prompt.json")
+    parser.add_argument("--num-prompts", type=int, default=10)
+    parser.add_argument("--height", type=int, default=HEIGHT)
+    parser.add_argument("--width", type=int, default=WIDTH)
+    parser.add_argument("--num-inference-steps", type=int, default=NUM_INFERENCE_STEPS)
+    parser.add_argument("--guidance-scale", type=float, default=GUIDANCE_SCALE)
+    parser.add_argument("--seed", type=int, default=SEED)
+    parser.add_argument("--output-dir", type=str, default="benchmarks/glm_image/vllm-omni/outputs")
+    parser.add_argument("--output-file", type=str, default=None, help="JSON file for metrics")
+    parser.add_argument("--stage-init-timeout", type=int, default=600)
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler for stage-level timing",
+    )
+    parser.add_argument(
+        "--enable-ar-profiler",
+        action="store_true",
+        help="Enable AR stage profiler to include AR timing in stage_durations",
+    )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        help="Enable detailed per-request pipeline stats logging",
+    )
+    args = parser.parse_args()
+    benchmark(args)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/qwen3-tts/README.md

@@ -1,107 +0,0 @@
-# Qwen3-TTS Benchmark
-
-Benchmarks for Qwen3-TTS text-to-speech models, comparing vLLM-Omni streaming serving against HuggingFace Transformers offline inference.
-
-## Prerequisites
-
-```bash
-pip install matplotlib aiohttp soundfile numpy tqdm
-pip install qwen_tts  # for HF baseline
-```
-
-## Quick Start
-
-Run the full benchmark (vllm-omni + HF baseline) with a single command:
-
-```bash
-cd benchmarks/qwen3-tts
-bash run_benchmark.sh
-```
-
-Results (JSON + PNG plots) are saved to `results/`.
-
-### Common options
-
-```bash
-# Only vllm-omni (skip HF baseline)
-bash run_benchmark.sh --async-only
-
-# Only HF baseline
-bash run_benchmark.sh --hf-only
-
-# Use a different model (e.g. 1.7B)
-MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
-
-# Use a Voice Clone model
-MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
-
-# Use batch size 16 for higher throughput
-BATCH_SIZE=16 bash run_benchmark.sh --async-only
-
-# Custom GPU, prompt count, concurrency levels
-GPU_DEVICE=1 NUM_PROMPTS=20 CONCURRENCY="1 4" bash run_benchmark.sh
-```
-
-## Manual Steps
-
-### 1) Start the vLLM-Omni server
-
-```bash
-CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
-    "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" \
-    --omni --host 127.0.0.1 --port 8000 \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --stage-overrides '{"0":{"max_num_seqs":1,"gpu_memory_utilization":0.3,"max_num_batched_tokens":512},"1":{"max_num_seqs":1,"gpu_memory_utilization":0.3,"max_num_batched_tokens":8192}}' \
-    --trust-remote-code
-```
-
-### 2) Run online serving benchmark
-
-```bash
-python benchmarks/qwen3-tts/vllm_omni/bench_tts_serve.py \
-    --port 8000 \
-    --num-prompts 50 \
-    --max-concurrency 1 4 10 \
-    --config-name "async_chunk" \
-    --result-dir results/
-```
-
-### 3) Run HuggingFace baseline
-
-```bash
-python benchmarks/qwen3-tts/transformers/bench_tts_hf.py \
-    --model "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" \
-    --num-prompts 50 \
-    --gpu-device 0 \
-    --result-dir results/
-```
-
-### 4) Generate comparison plots
-
-```bash
-python benchmarks/qwen3-tts/plot_results.py \
-    --results results/bench_async_chunk_*.json results/bench_hf_transformers_*.json \
-    --labels "vllm-omni" "hf_transformers" \
-    --output results/comparison.png
-```
-
-## Batch-size presets
-
-The bench script loads the bundled production deploy (`vllm_omni/deploy/qwen3_tts.yaml`) and layers per-stage budgets on top via `--stage-overrides`, driven by the `BATCH_SIZE` env var. Each batch size picks compatible per-stage `max_num_seqs`, `max_num_batched_tokens`, and `gpu_memory_utilization` defaults:
-
-| `BATCH_SIZE` | Description |
-|:--:|-------------|
-| `1` (default) | Single-request processing (lowest latency) |
-| `4`  | Moderate-throughput concurrent processing |
-| `16` | High-throughput concurrent processing |
-
-The 2-stage pipeline (Talker -> Code2Wav) runs with `async_chunk` streaming enabled via the prod deploy; the `SharedMemoryConnector` streams codec frames (25-frame chunks with 25-frame context overlap) between stages.
-
-The model is specified via the CLI `--model` flag (or `MODEL` env var), so the same bench script works for both the 0.6B and 1.7B model variants.
-
-## Metrics
-
-- **TTFP (Time to First Audio Packet)**: Time from request to first audio chunk (streaming latency)
-- **E2E (End-to-End Latency)**: Total time from request to complete audio response
-- **RTF (Real-Time Factor)**: E2E latency / audio duration. RTF < 1.0 means faster-than-real-time synthesis
-- **Throughput**: Total audio seconds generated per wall-clock second

benchmarks/qwen3-tts/plot_results.py

@@ -1,254 +0,0 @@
-"""Plot Qwen3-TTS benchmark results.
-
-Generates comparison bar charts similar to the async_chunk design doc:
-- TTFP (Time-to-First-Packet) across concurrency levels
-- E2E latency across concurrency levels
-- RTF (Real-Time Factor) across concurrency levels
-
-Usage:
-    # Compare two configs (async_chunk vs no_async_chunk):
-    python plot_results.py \
-        --results results/bench_async_chunk_*.json results/bench_no_async_chunk_*.json \
-        --labels "async_chunk" "no_async_chunk" \
-        --output results/qwen3_tts_benchmark.png
-
-    # Single config:
-    python plot_results.py \
-        --results results/bench_async_chunk_*.json \
-        --labels "async_chunk" \
-        --output results/qwen3_tts_benchmark.png
-"""
-
-import argparse
-import json
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import numpy as np
-
-
-def load_results(result_files: list[str]) -> list[list[dict]]:
-    """Load benchmark results from JSON files."""
-    all_results = []
-    for f in result_files:
-        with open(f) as fh:
-            data = json.load(fh)
-        all_results.append(data)
-    return all_results
-
-
-def plot_comparison(
-    all_results: list[list[dict]],
-    labels: list[str],
-    output_path: str,
-    title_prefix: str = "Qwen3-TTS",
-):
-    """Generate comparison bar charts."""
-    n_configs = len(all_results)
-
-    # Collect concurrency levels present in ALL configs (skip missing data)
-    all_concurrencies = [set(r["concurrency"] for r in results) for results in all_results]
-    concurrencies = sorted(set.union(*all_concurrencies))
-
-    # Build data arrays, using None for missing concurrency levels
-    ttfp_data = {label: [] for label in labels}
-    e2e_data = {label: [] for label in labels}
-    rtf_data = {label: [] for label in labels}
-    throughput_data = {label: [] for label in labels}
-
-    for results, label in zip(all_results, labels):
-        conc_map = {r["concurrency"]: r for r in results}
-        for c in concurrencies:
-            r = conc_map.get(c)
-            ttfp_data[label].append(r["mean_ttfp_ms"] if r else None)
-            e2e_data[label].append(r["mean_e2e_ms"] if r else None)
-            rtf_data[label].append(r["mean_rtf"] if r else None)
-            throughput_data[label].append(r["audio_throughput"] if r else None)
-
-    fig, axes = plt.subplots(2, 2, figsize=(14, 10))
-    fig.suptitle(f"{title_prefix} Performance Benchmark", fontsize=16, fontweight="bold")
-
-    x = np.arange(len(concurrencies))
-    width = 0.35 if n_configs == 2 else 0.5
-    if n_configs > 1:
-        offsets = np.linspace(-width / 2 * (n_configs - 1), width / 2 * (n_configs - 1), n_configs)
-    else:
-        offsets = [0]
-
-    colors = ["#2196F3", "#FF5722", "#4CAF50", "#FFC107"]
-
-    def plot_metric(ax, data_dict, ylabel, title, fmt=".1f"):
-        bars = []
-        for i, (label, values) in enumerate(data_dict.items()):
-            # Replace None with 0 for plotting, but track which are missing
-            plot_values = [v if v is not None else 0 for v in values]
-            color = colors[i % len(colors)]
-            bar = ax.bar(x + offsets[i], plot_values, width, label=label, color=color, alpha=0.85)
-            bars.append(bar)
-            # Add value labels on bars (skip None/missing data)
-            max_val = max((v for v in values if v is not None), default=1)
-            for rect, val in zip(bar, values):
-                if val is not None and val > 0:
-                    ax.text(
-                        rect.get_x() + rect.get_width() / 2,
-                        rect.get_height() + max_val * 0.02,
-                        f"{val:{fmt}}",
-                        ha="center",
-                        va="bottom",
-                        fontsize=9,
-                        fontweight="bold",
-                    )
-        ax.set_xlabel("Concurrency", fontsize=12)
-        ax.set_ylabel(ylabel, fontsize=12)
-        ax.set_title(title, fontsize=13, fontweight="bold")
-        ax.set_xticks(x)
-        ax.set_xticklabels([str(c) for c in concurrencies])
-        ax.legend(fontsize=10)
-        ax.grid(axis="y", alpha=0.3)
-        ax.set_axisbelow(True)
-
-    plot_metric(axes[0, 0], ttfp_data, "TTFP (ms)", "Time to First Audio Packet (TTFP)")
-    plot_metric(axes[0, 1], e2e_data, "E2E Latency (ms)", "End-to-End Latency (E2E)")
-    plot_metric(axes[1, 0], rtf_data, "RTF", "Real-Time Factor (RTF)", fmt=".3f")
-    plot_metric(axes[1, 1], throughput_data, "Audio-sec / Wall-sec", "Audio Throughput", fmt=".2f")
-
-    plt.tight_layout()
-    plt.savefig(output_path, dpi=150, bbox_inches="tight")
-    print(f"Plot saved to {output_path}")
-    plt.close()
-
-
-def plot_single_summary(results: list[dict], label: str, output_path: str):
-    """Generate a single-config summary with percentile breakdown."""
-    concurrencies = [r["concurrency"] for r in results]
-
-    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
-    fig.suptitle(f"Qwen3-TTS Benchmark - {label}", fontsize=15, fontweight="bold")
-
-    # TTFP breakdown
-    ax = axes[0]
-    means = [r["mean_ttfp_ms"] for r in results]
-    medians = [r["median_ttfp_ms"] for r in results]
-    p90s = [r["p90_ttfp_ms"] for r in results]
-    p99s = [r["p99_ttfp_ms"] for r in results]
-    x = np.arange(len(concurrencies))
-    w = 0.2
-    ax.bar(x - 1.5 * w, means, w, label="mean", color="#2196F3")
-    ax.bar(x - 0.5 * w, medians, w, label="median", color="#4CAF50")
-    ax.bar(x + 0.5 * w, p90s, w, label="p90", color="#FF9800")
-    ax.bar(x + 1.5 * w, p99s, w, label="p99", color="#F44336")
-    ax.set_xticks(x)
-    ax.set_xticklabels([str(c) for c in concurrencies])
-    ax.set_xlabel("Concurrency")
-    ax.set_ylabel("TTFP (ms)")
-    ax.set_title("Time to First Audio Packet")
-    ax.legend(fontsize=9)
-    ax.grid(axis="y", alpha=0.3)
-
-    # E2E breakdown
-    ax = axes[1]
-    means = [r["mean_e2e_ms"] for r in results]
-    medians = [r["median_e2e_ms"] for r in results]
-    p90s = [r["p90_e2e_ms"] for r in results]
-    p99s = [r["p99_e2e_ms"] for r in results]
-    ax.bar(x - 1.5 * w, means, w, label="mean", color="#2196F3")
-    ax.bar(x - 0.5 * w, medians, w, label="median", color="#4CAF50")
-    ax.bar(x + 0.5 * w, p90s, w, label="p90", color="#FF9800")
-    ax.bar(x + 1.5 * w, p99s, w, label="p99", color="#F44336")
-    ax.set_xticks(x)
-    ax.set_xticklabels([str(c) for c in concurrencies])
-    ax.set_xlabel("Concurrency")
-    ax.set_ylabel("E2E Latency (ms)")
-    ax.set_title("End-to-End Latency")
-    ax.legend(fontsize=9)
-    ax.grid(axis="y", alpha=0.3)
-
-    # RTF
-    ax = axes[2]
-    means = [r["mean_rtf"] for r in results]
-    medians = [r["median_rtf"] for r in results]
-    ax.bar(x - 0.15, means, 0.3, label="mean", color="#2196F3")
-    ax.bar(x + 0.15, medians, 0.3, label="median", color="#4CAF50")
-    ax.set_xticks(x)
-    ax.set_xticklabels([str(c) for c in concurrencies])
-    ax.set_xlabel("Concurrency")
-    ax.set_ylabel("RTF")
-    ax.set_title("Real-Time Factor")
-    ax.legend(fontsize=9)
-    ax.grid(axis="y", alpha=0.3)
-
-    plt.tight_layout()
-    plt.savefig(output_path, dpi=150, bbox_inches="tight")
-    print(f"Plot saved to {output_path}")
-    plt.close()
-
-
-def print_comparison_table(all_results: list[list[dict]], labels: list[str]):
-    """Print a markdown-formatted comparison table."""
-    concurrencies = sorted(set(r["concurrency"] for r in all_results[0]))
-
-    print("\n## Benchmark Results\n")
-    header = "| Metric | Concurrency |"
-    sep = "| --- | --- |"
-    for label in labels:
-        header += f" {label} |"
-        sep += " --- |"
-    print(header)
-    print(sep)
-
-    for metric, key, fmt in [
-        ("TTFP (ms)", "mean_ttfp_ms", ".1f"),
-        ("E2E (ms)", "mean_e2e_ms", ".1f"),
-        ("RTF", "mean_rtf", ".3f"),
-        ("Throughput (audio-s/s)", "audio_throughput", ".2f"),
-    ]:
-        for c in concurrencies:
-            row = f"| {metric} | {c} |"
-            for results in all_results:
-                conc_map = {r["concurrency"]: r for r in results}
-                val = conc_map.get(c, {}).get(key, 0)
-                row += f" {val:{fmt}} |"
-            print(row)
-
-    # Improvement calculation (only if 2 configs)
-    if len(all_results) == 2:
-        print(f"\n## Improvement ({labels[0]} vs {labels[1]})\n")
-        print("| Metric | Concurrency | Improvement |")
-        print("| --- | --- | --- |")
-        for metric, key in [("TTFP", "mean_ttfp_ms"), ("E2E", "mean_e2e_ms"), ("RTF", "mean_rtf")]:
-            for c in concurrencies:
-                m0 = {r["concurrency"]: r for r in all_results[0]}
-                m1 = {r["concurrency"]: r for r in all_results[1]}
-                v0 = m0.get(c, {}).get(key, 0)
-                v1 = m1.get(c, {}).get(key, 0)
-                if v1 > 0:
-                    pct = (v1 - v0) / v1 * 100
-                    print(f"| {metric} | {c} | {pct:+.1f}% |")
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Plot Qwen3-TTS benchmark results")
-    parser.add_argument(
-        "--results", type=str, nargs="+", required=True, help="Path(s) to result JSON files (one per config)"
-    )
-    parser.add_argument(
-        "--labels", type=str, nargs="+", required=True, help="Labels for each config (must match --results count)"
-    )
-    parser.add_argument("--output", type=str, default="results/qwen3_tts_benchmark.png", help="Output image path")
-    parser.add_argument("--title", type=str, default="Qwen3-TTS", help="Title prefix for the plot")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    assert len(args.results) == len(args.labels), "--results and --labels must have the same count"
-
-    all_results = load_results(args.results)
-    print_comparison_table(all_results, args.labels)
-
-    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
-
-    if len(all_results) == 1:
-        plot_single_summary(all_results[0], args.labels[0], args.output)
-    else:
-        plot_comparison(all_results, args.labels, args.output, title_prefix=args.title)

benchmarks/qwen3-tts/results/.gitignore

@@ -1,3 +0,0 @@
-# Benchmark results are machine-specific - do not commit
-*
-!.gitignore

benchmarks/qwen3-tts/run_benchmark.sh

@@ -1,285 +0,0 @@
-#!/bin/bash
-# Qwen3-TTS Benchmark Runner
-#
-# Compares vllm-omni streaming serving vs HuggingFace transformers offline inference.
-# Produces JSON results and comparison plots.
-#
-# Usage:
-#   # Full comparison (vllm-omni + HF):
-#   bash run_benchmark.sh
-#
-#   # Only vllm-omni async_chunk config:
-#   bash run_benchmark.sh --async-only
-#
-#   # Only HuggingFace baseline:
-#   bash run_benchmark.sh --hf-only
-#
-#   # vllm-omni only (skip HF):
-#   bash run_benchmark.sh --skip-hf
-#
-#   # Custom settings:
-#   GPU_DEVICE=1 NUM_PROMPTS=20 CONCURRENCY="1 4" bash run_benchmark.sh
-#
-#   # Use 1.7B model:
-#   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
-#
-#   # Use Voice Clone model
-#   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
-#
-#   # Use batch_size=4:
-#   BATCH_SIZE=4 bash run_benchmark.sh --async-only
-#
-# Environment variables:
-#   GPU_DEVICE       - GPU index to use (default: 0)
-#   NUM_PROMPTS      - Number of prompts per concurrency level (default: 50)
-#   CONCURRENCY      - Space-separated concurrency levels (default: "1 4 10")
-#   MODEL            - Model name (default: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice)
-#   PORT             - Server port (default: 8000)
-#   BATCH_SIZE       - Per-stage ``max_num_seqs`` for both talker and code2wav (default: 1)
-#   GPU_MEM_TALKER   - gpu_memory_utilization for talker stage (default: 0.3 at bs=1, else 0.2)
-#   GPU_MEM_CODE2WAV - gpu_memory_utilization for code2wav stage (default: 0.3 at bs=1, else 0.2)
-#   TASK_TYPE        - Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
-
-# Defaults
-GPU_DEVICE="${GPU_DEVICE:-0}"
-NUM_PROMPTS="${NUM_PROMPTS:-50}"
-CONCURRENCY="${CONCURRENCY:-1 4 10}"
-MODEL="${MODEL:-Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice}"
-PORT="${PORT:-8000}"
-BATCH_SIZE="${BATCH_SIZE:-1}"
-DEFAULT_MEM=$([ "${BATCH_SIZE}" = "1" ] && echo "0.3" || echo "0.2")
-GPU_MEM_TALKER="${GPU_MEM_TALKER:-${DEFAULT_MEM}}"
-GPU_MEM_CODE2WAV="${GPU_MEM_CODE2WAV:-${DEFAULT_MEM}}"
-NUM_WARMUPS="${NUM_WARMUPS:-3}"
-DEPLOY_CONFIG="vllm_omni/deploy/qwen3_tts.yaml"
-RESULT_DIR="${SCRIPT_DIR}/results"
-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
-TASK_TYPE="${TASK_TYPE:-CustomVoice}"
-
-# Build --stage-overrides JSON from BATCH_SIZE + GPU_MEM_*.
-STAGE_OVERRIDES=$(
-    BATCH_SIZE="${BATCH_SIZE}" \
-    GPU_MEM_TALKER="${GPU_MEM_TALKER}" \
-    GPU_MEM_CODE2WAV="${GPU_MEM_CODE2WAV}" \
-    python - <<'PYEOF'
-import json, os
-bs = int(os.environ["BATCH_SIZE"])
-mem_t = float(os.environ["GPU_MEM_TALKER"])
-mem_c = float(os.environ["GPU_MEM_CODE2WAV"])
-# Prefill budget grows with batch size on both stages.
-talker_batched = 512 if bs <= 4 else 4096
-code2wav_batched = 8192 if bs <= 4 else 32768
-print(json.dumps({
-    "0": {"max_num_seqs": bs, "gpu_memory_utilization": mem_t, "max_num_batched_tokens": talker_batched},
-    "1": {"max_num_seqs": bs, "gpu_memory_utilization": mem_c, "max_num_batched_tokens": code2wav_batched},
-}))
-PYEOF
-)
-
-# Parse args
-RUN_ASYNC=true
-RUN_HF=true
-for arg in "$@"; do
-    case "$arg" in
-        --async-only) RUN_HF=false ;;
-        --hf-only) RUN_ASYNC=false ;;
-        --skip-hf) RUN_HF=false ;;
-    esac
-done
-
-mkdir -p "${RESULT_DIR}"
-
-echo "============================================================"
-echo " Qwen3-TTS Benchmark"
-echo "============================================================"
-echo " GPU:            ${GPU_DEVICE}"
-echo " Model:          ${MODEL}"
-echo " Prompts:        ${NUM_PROMPTS}"
-echo " Concurrency:    ${CONCURRENCY}"
-echo " Port:           ${PORT}"
-echo " Deploy config:  ${DEPLOY_CONFIG}"
-echo " Batch size:     ${BATCH_SIZE}"
-echo " GPU mem T/C:    ${GPU_MEM_TALKER} / ${GPU_MEM_CODE2WAV}"
-echo " Results:        ${RESULT_DIR}"
-echo " Task type:      ${TASK_TYPE}"
-echo "============================================================"
-
-# Start server and wait for it to be ready
-start_server() {
-    local config_name="$1"
-    local log_file="${RESULT_DIR}/server_${config_name}_${TIMESTAMP}.log"
-
-    echo ""
-    echo "Starting server with config: ${config_name}"
-    echo "  Deploy config: ${DEPLOY_CONFIG}"
-    echo "  Stage overrides: ${STAGE_OVERRIDES}"
-    echo "  Log file: ${log_file}"
-
-    VLLM_WORKER_MULTIPROC_METHOD=spawn \
-    CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
-    python -m vllm_omni.entrypoints.cli.main serve "${MODEL}" \
-        --omni \
-        --host 127.0.0.1 \
-        --port "${PORT}" \
-        --deploy-config "${DEPLOY_CONFIG}" \
-        --stage-overrides "${STAGE_OVERRIDES}" \
-        --stage-init-timeout 120 \
-        --trust-remote-code \
-        --disable-log-stats \
-        > "${log_file}" 2>&1 &
-
-    SERVER_PID=$!
-    echo "  Server PID: ${SERVER_PID}"
-
-    # Wait for server to be ready
-    echo "  Waiting for server to be ready..."
-    local max_wait=300
-    local waited=0
-    while [ ${waited} -lt ${max_wait} ]; do
-        if curl -sf "http://127.0.0.1:${PORT}/v1/models" > /dev/null 2>&1; then
-            echo "  Server is ready! (waited ${waited}s)"
-            return 0
-        fi
-        # Check if process is still alive
-        if ! kill -0 ${SERVER_PID} 2>/dev/null; then
-            echo "  ERROR: Server process died. Check log: ${log_file}"
-            tail -20 "${log_file}"
-            return 1
-        fi
-        sleep 2
-        waited=$((waited + 2))
-    done
-
-    echo "  ERROR: Server did not start within ${max_wait}s. Check log: ${log_file}"
-    kill ${SERVER_PID} 2>/dev/null || true
-    return 1
-}
-
-# Stop the server
-stop_server() {
-    if [ -n "${SERVER_PID:-}" ]; then
-        echo "  Stopping server (PID: ${SERVER_PID})..."
-        kill ${SERVER_PID} 2>/dev/null || true
-        wait ${SERVER_PID} 2>/dev/null || true
-        # Kill any remaining child processes on the port
-        local pids
-        pids=$(lsof -ti:${PORT} 2>/dev/null || true)
-        if [ -n "${pids}" ]; then
-            echo "  Cleaning up remaining processes on port ${PORT}..."
-            echo "${pids}" | xargs kill -9 2>/dev/null || true
-        fi
-        echo "  Server stopped."
-        SERVER_PID=""
-    fi
-}
-
-# Cleanup on exit
-trap 'stop_server' EXIT
-
-# Run benchmark for a given config
-run_bench() {
-    local config_name="$1"
-
-    echo ""
-    echo "============================================================"
-    echo " Benchmarking: ${config_name}"
-    echo "============================================================"
-
-    start_server "${config_name}"
-
-    # Convert concurrency string to args
-    local conc_args=""
-    for c in ${CONCURRENCY}; do
-        conc_args="${conc_args} ${c}"
-    done
-
-    cd "${PROJECT_ROOT}"
-    python "${SCRIPT_DIR}/vllm_omni/bench_tts_serve.py" \
-        --host 127.0.0.1 \
-        --port "${PORT}" \
-        --num-prompts "${NUM_PROMPTS}" \
-        --max-concurrency ${conc_args} \
-        --num-warmups "${NUM_WARMUPS}" \
-        --config-name "${config_name}" \
-        --result-dir "${RESULT_DIR}" \
-        --task-type "${TASK_TYPE}"
-
-    stop_server
-
-    # Allow GPU memory to settle
-    sleep 5
-}
-
-# Run vllm-omni benchmark
-if [ "${RUN_ASYNC}" = true ]; then
-    run_bench "async_chunk"
-fi
-
-# Run HuggingFace baseline benchmark
-if [ "${RUN_HF}" = true ]; then
-    echo ""
-    echo "============================================================"
-    echo " Benchmarking: HuggingFace transformers (offline)"
-    echo "============================================================"
-
-    cd "${PROJECT_ROOT}"
-    python "${SCRIPT_DIR}/transformers/bench_tts_hf.py" \
-        --model "${MODEL}" \
-        --num-prompts "${NUM_PROMPTS}" \
-        --num-warmups "${NUM_WARMUPS}" \
-        --gpu-device "${GPU_DEVICE}" \
-        --config-name "hf_transformers" \
-        --result-dir "${RESULT_DIR}" \
-        --task-type "${TASK_TYPE}"
-
-    # Allow GPU memory to settle
-    sleep 5
-fi
-
-# Plot results
-echo ""
-echo "============================================================"
-echo " Generating plots..."
-echo "============================================================"
-
-RESULT_FILES=""
-LABELS=""
-
-if [ "${RUN_ASYNC}" = true ]; then
-    ASYNC_FILE=$(ls -t "${RESULT_DIR}"/bench_async_chunk_*.json 2>/dev/null | head -1)
-    if [ -n "${ASYNC_FILE}" ]; then
-        RESULT_FILES="${ASYNC_FILE}"
-        LABELS="async_chunk"
-    fi
-fi
-
-if [ "${RUN_HF}" = true ]; then
-    HF_FILE=$(ls -t "${RESULT_DIR}"/bench_hf_transformers_*.json 2>/dev/null | head -1)
-    if [ -n "${HF_FILE}" ]; then
-        if [ -n "${RESULT_FILES}" ]; then
-            RESULT_FILES="${RESULT_FILES} ${HF_FILE}"
-            LABELS="${LABELS} hf_transformers"
-        else
-            RESULT_FILES="${HF_FILE}"
-            LABELS="hf_transformers"
-        fi
-    fi
-fi
-
-if [ -n "${RESULT_FILES}" ]; then
-    python "${SCRIPT_DIR}/plot_results.py" \
-        --results ${RESULT_FILES} \
-        --labels ${LABELS} \
-        --output "${RESULT_DIR}/qwen3_tts_benchmark_${TIMESTAMP}.png"
-fi
-
-echo ""
-echo "============================================================"
-echo " Benchmark complete!"
-echo " Results: ${RESULT_DIR}"
-echo "============================================================"

benchmarks/qwen3-tts/transformers/bench_tts_hf.py

@@ -1,301 +0,0 @@
-"""Benchmark Qwen3-TTS using HuggingFace transformers (qwen_tts library).
-
-Measures E2E latency, RTF, and audio duration for offline (non-serving) inference.
-Results are saved in the same JSON format as bench_tts_serve.py for unified plotting.
-
-Usage:
-    python bench_tts_hf.py \
-        --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \
-        --num-prompts 50 \
-        --num-warmups 3 \
-        --gpu-device 0 \
-        --result-dir results/
-"""
-
-import argparse
-import json
-import time
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from pathlib import Path
-
-import numpy as np
-import soundfile as sf
-import torch
-
-PROMPTS = [
-    "Hello, welcome to the voice synthesis benchmark test.",
-    "She said she would be here by noon, but nobody showed up.",
-    "The quick brown fox jumps over the lazy dog near the riverbank.",
-    "I can't believe how beautiful the sunset looks from up here on the mountain.",
-    "Please remember to bring your identification documents to the appointment tomorrow morning.",
-    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
-    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
-    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
-    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
-    "The train leaves at half past seven, so we need to arrive at the station before then.",
-    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
-    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
-]
-
-REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
-
-
-@dataclass
-class BenchmarkResult:
-    config_name: str = ""
-    concurrency: int = 1  # always 1 for offline
-    num_prompts: int = 0
-    completed: int = 0
-    failed: int = 0
-    duration_s: float = 0.0
-    # TTFP stats - not applicable for HF offline, set to E2E for compatibility
-    mean_ttfp_ms: float = 0.0
-    median_ttfp_ms: float = 0.0
-    std_ttfp_ms: float = 0.0
-    p90_ttfp_ms: float = 0.0
-    p95_ttfp_ms: float = 0.0
-    p99_ttfp_ms: float = 0.0
-    # E2E stats (ms)
-    mean_e2e_ms: float = 0.0
-    median_e2e_ms: float = 0.0
-    std_e2e_ms: float = 0.0
-    p90_e2e_ms: float = 0.0
-    p95_e2e_ms: float = 0.0
-    p99_e2e_ms: float = 0.0
-    # RTF stats
-    mean_rtf: float = 0.0
-    median_rtf: float = 0.0
-    std_rtf: float = 0.0
-    p99_rtf: float = 0.0
-    # Audio stats
-    mean_audio_duration_s: float = 0.0
-    total_audio_duration_s: float = 0.0
-    audio_throughput: float = 0.0
-    request_throughput: float = 0.0
-    # Per-request details
-    per_request: list = field(default_factory=list)
-
-
-def generate_audio(model, prompt: str, args):
-    if args.task_type == "Base":
-        return model.generate_voice_clone(
-            text=prompt,
-            language=args.language,
-            ref_audio=REF_AUDIO,
-            ref_text=REF_TEXT,
-        )
-
-    if args.task_type == "VoiceDesign":
-        return model.generate_voice_design(
-            text=prompt,
-            language=args.language,
-            instruct=INSTRUCT,
-        )
-
-    return model.generate_custom_voice(
-        text=prompt,
-        language=args.language,
-        speaker=args.voice,
-    )
-
-
-def run_benchmark(args):
-    from qwen_tts import Qwen3TTSModel
-
-    device = f"cuda:{args.gpu_device}"
-    print(f"Loading model: {args.model} on {device}")
-    model = Qwen3TTSModel.from_pretrained(
-        args.model,
-        device_map=device,
-        dtype=torch.bfloat16,
-    )
-    print("Model loaded.")
-
-    # Build prompt list
-    prompts = [PROMPTS[i % len(PROMPTS)] for i in range(args.num_prompts)]
-
-    # Warmup
-    if args.num_warmups > 0:
-        print(f"Warming up with {args.num_warmups} requests...")
-        for i in range(args.num_warmups):
-            p = PROMPTS[i % len(PROMPTS)]
-            wavs, sr = generate_audio(model, p, args)
-        # Sync GPU
-        torch.cuda.synchronize(device)
-        print("Warmup done.")
-
-    # Benchmark
-    print(f"Running {args.num_prompts} requests sequentially...")
-    e2e_times = []
-    rtfs = []
-    audio_durations = []
-    per_request = []
-    failed = 0
-
-    audio_dir = None
-    if args.save_audio:
-        audio_dir = Path(args.result_dir) / "audio_hf"
-        audio_dir.mkdir(parents=True, exist_ok=True)
-
-    total_start = time.perf_counter()
-
-    for i, prompt in enumerate(prompts):
-        try:
-            torch.cuda.synchronize(device)
-            st = time.perf_counter()
-
-            wavs, sr = generate_audio(model, prompt, args)
-
-            torch.cuda.synchronize(device)
-            elapsed = time.perf_counter() - st
-
-            # Compute audio duration
-            audio_samples = wavs[0]
-            if isinstance(audio_samples, torch.Tensor):
-                audio_samples = audio_samples.cpu().numpy()
-            audio_dur = len(audio_samples) / sr
-
-            rtf = elapsed / audio_dur if audio_dur > 0 else 0.0
-
-            e2e_times.append(elapsed)
-            rtfs.append(rtf)
-            audio_durations.append(audio_dur)
-            per_request.append(
-                {
-                    "e2e_ms": elapsed * 1000,
-                    "ttfp_ms": elapsed * 1000,  # no streaming, TTFP = E2E
-                    "rtf": rtf,
-                    "audio_duration_s": audio_dur,
-                    "prompt": prompt,
-                }
-            )
-
-            if audio_dir:
-                sf.write(str(audio_dir / f"output_{i:04d}.wav"), audio_samples, sr)
-
-            if (i + 1) % 10 == 0 or i == 0:
-                print(
-                    f"  [{i + 1}/{args.num_prompts}] e2e={elapsed * 1000:.0f}ms  rtf={rtf:.3f}  audio={audio_dur:.2f}s"
-                )
-
-        except Exception as e:
-            print(f"  [{i + 1}/{args.num_prompts}] FAILED: {e}")
-            failed += 1
-
-    total_duration = time.perf_counter() - total_start
-    completed = len(e2e_times)
-
-    # Compute stats
-    result = BenchmarkResult(
-        config_name=args.config_name,
-        concurrency=1,
-        num_prompts=args.num_prompts,
-        completed=completed,
-        failed=failed,
-        duration_s=total_duration,
-    )
-
-    if e2e_times:
-        e2e_ms = [t * 1000 for t in e2e_times]
-
-        result.mean_e2e_ms = float(np.mean(e2e_ms))
-        result.median_e2e_ms = float(np.median(e2e_ms))
-        result.std_e2e_ms = float(np.std(e2e_ms))
-        result.p90_e2e_ms = float(np.percentile(e2e_ms, 90))
-        result.p95_e2e_ms = float(np.percentile(e2e_ms, 95))
-        result.p99_e2e_ms = float(np.percentile(e2e_ms, 99))
-
-        # For HF offline, TTFP = E2E (no streaming)
-        result.mean_ttfp_ms = result.mean_e2e_ms
-        result.median_ttfp_ms = result.median_e2e_ms
-        result.std_ttfp_ms = result.std_e2e_ms
-        result.p90_ttfp_ms = result.p90_e2e_ms
-        result.p95_ttfp_ms = result.p95_e2e_ms
-        result.p99_ttfp_ms = result.p99_e2e_ms
-
-        result.mean_rtf = float(np.mean(rtfs))
-        result.median_rtf = float(np.median(rtfs))
-        result.std_rtf = float(np.std(rtfs))
-        result.p99_rtf = float(np.percentile(rtfs, 99))
-
-        result.mean_audio_duration_s = float(np.mean(audio_durations))
-        result.total_audio_duration_s = float(np.sum(audio_durations))
-        result.audio_throughput = result.total_audio_duration_s / total_duration
-        result.request_throughput = completed / total_duration
-        result.per_request = per_request
-
-    # Print summary in standardized performance template
-    W = 50
-    print("")
-    print(f"{'=' * W}")
-    print(f"{'Serving Benchmark Result':^{W}}")
-    print(f"{'=' * W}")
-    print(f"{'Successful requests:':<40}{completed:<10}")
-    print(f"{'Failed requests:':<40}{failed:<10}")
-    print(f"{'Maximum request concurrency:':<40}{1:<10}")
-    print(f"{'Benchmark duration (s):':<40}{total_duration:<10.2f}")
-    print(f"{'Request throughput (req/s):':<40}{result.request_throughput:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'End-to-end Latency':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean E2EL (ms):':<40}{result.mean_e2e_ms:<10.2f}")
-    print(f"{'Median E2EL (ms):':<40}{result.median_e2e_ms:<10.2f}")
-    print(f"{'P99 E2EL (ms):':<40}{result.p99_e2e_ms:<10.2f}")
-    print(f"{'=' * W}")
-    print(f"{'Audio Result':^{W}}")
-    print(f"{'=' * W}")
-    print(f"{'Total audio duration generated (s):':<40}{result.total_audio_duration_s:<10.2f}")
-    print(f"{'Audio throughput (audio duration/s):':<40}{result.audio_throughput:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'Time to First Packet':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean AUDIO_TTFP (ms):':<40}{result.mean_ttfp_ms:<10.2f}")
-    print(f"{'Median AUDIO_TTFP (ms):':<40}{result.median_ttfp_ms:<10.2f}")
-    print(f"{'P99 AUDIO_TTFP (ms):':<40}{result.p99_ttfp_ms:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'Real Time Factor':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean AUDIO_RTF:':<40}{result.mean_rtf:<10.3f}")
-    print(f"{'Median AUDIO_RTF:':<40}{result.median_rtf:<10.3f}")
-    print(f"{'P99 AUDIO_RTF:':<40}{result.p99_rtf:<10.3f}")
-    print(f"{'=' * W}")
-    print("")
-
-    # Save results (as a list with single concurrency=1 entry, matching serve format)
-    result_dir = Path(args.result_dir)
-    result_dir.mkdir(parents=True, exist_ok=True)
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
-
-    with open(result_file, "w") as f:
-        json.dump([asdict(result)], f, indent=2)
-    print(f"Results saved to {result_file}")
-
-    return result
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Qwen3-TTS HuggingFace Benchmark")
-    parser.add_argument(
-        "--model", type=str, default="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", help="HuggingFace model name or path"
-    )
-    parser.add_argument("--num-prompts", type=int, default=50)
-    parser.add_argument("--num-warmups", type=int, default=3)
-    parser.add_argument("--gpu-device", type=int, default=0)
-    parser.add_argument("--voice", type=str, default="Vivian")
-    parser.add_argument("--language", type=str, default="English")
-    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
-    parser.add_argument(
-        "--config-name", type=str, default="hf_transformers", help="Label for this config (used in filenames)"
-    )
-    parser.add_argument("--result-dir", type=str, default="results")
-    parser.add_argument("--save-audio", action="store_true", help="Save generated audio files")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    run_benchmark(args)

benchmarks/qwen3-tts/vllm_omni/bench_async_chunk.py

@@ -1,301 +0,0 @@
-"""Benchmark comparing async_chunk on vs off for Qwen3-TTS.
-
-Measures TTFP (Time-to-First-Packet), E2E latency, and RTF across
-concurrency levels for both async_chunk modes. Saves results as JSON.
-
-Usage:
-    # Run against a server already serving with a given config:
-    python bench_async_chunk.py \
-        --host 127.0.0.1 --port 8000 \
-        --config-name async_chunk_on \
-        --num-prompts 50 \
-        --max-concurrency 1 10 \
-        --result-dir results/
-"""
-
-import argparse
-import asyncio
-import json
-import time
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from pathlib import Path
-
-import aiohttp
-import numpy as np
-from tqdm.asyncio import tqdm
-
-PROMPTS = [
-    "Hello, welcome to the voice synthesis benchmark test.",
-    "She said she would be here by noon, but nobody showed up.",
-    "The quick brown fox jumps over the lazy dog near the riverbank.",
-    "I can't believe how beautiful the sunset looks from up here on the mountain.",
-    "Please remember to bring your identification documents to the appointment tomorrow morning.",
-    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
-    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
-    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
-    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
-    "The train leaves at half past seven, so we need to arrive at the station before then.",
-    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
-    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
-]
-
-
-@dataclass
-class RequestResult:
-    success: bool = False
-    ttfp: float = 0.0
-    e2e: float = 0.0
-    audio_bytes: int = 0
-    audio_duration: float = 0.0
-    rtf: float = 0.0
-    prompt: str = ""
-    error: str = ""
-
-
-@dataclass
-class BenchmarkResult:
-    config_name: str = ""
-    concurrency: int = 0
-    num_prompts: int = 0
-    completed: int = 0
-    failed: int = 0
-    duration_s: float = 0.0
-    mean_ttfp_ms: float = 0.0
-    median_ttfp_ms: float = 0.0
-    std_ttfp_ms: float = 0.0
-    p90_ttfp_ms: float = 0.0
-    p95_ttfp_ms: float = 0.0
-    p99_ttfp_ms: float = 0.0
-    mean_e2e_ms: float = 0.0
-    median_e2e_ms: float = 0.0
-    std_e2e_ms: float = 0.0
-    p90_e2e_ms: float = 0.0
-    p95_e2e_ms: float = 0.0
-    p99_e2e_ms: float = 0.0
-    mean_rtf: float = 0.0
-    median_rtf: float = 0.0
-    std_rtf: float = 0.0
-    mean_audio_duration_s: float = 0.0
-    total_audio_duration_s: float = 0.0
-    audio_throughput: float = 0.0
-    request_throughput: float = 0.0
-    per_request: list = field(default_factory=list)
-
-
-def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = 24000, sample_width: int = 2) -> float:
-    return num_bytes / sample_width / sample_rate
-
-
-async def send_tts_request(
-    session: aiohttp.ClientSession,
-    api_url: str,
-    prompt: str,
-    voice: str = "vivian",
-    language: str = "English",
-    stream: bool = True,
-    pbar: tqdm | None = None,
-) -> RequestResult:
-    payload = {
-        "input": prompt,
-        "voice": voice,
-        "language": language,
-        "stream": stream,
-        "response_format": "pcm",
-    }
-
-    result = RequestResult(prompt=prompt)
-    st = time.perf_counter()
-
-    try:
-        async with session.post(api_url, json=payload) as response:
-            if response.status != 200:
-                result.error = f"HTTP {response.status}: {await response.text()}"
-                return result
-
-            first_chunk = True
-            total_bytes = 0
-
-            async for chunk in response.content.iter_any():
-                if first_chunk and len(chunk) > 0:
-                    result.ttfp = time.perf_counter() - st
-                    first_chunk = False
-                total_bytes += len(chunk)
-
-            result.e2e = time.perf_counter() - st
-            result.audio_bytes = total_bytes
-            result.audio_duration = pcm_bytes_to_duration(total_bytes)
-            if result.audio_duration > 0:
-                result.rtf = result.e2e / result.audio_duration
-            result.success = True
-
-    except Exception as e:
-        result.error = str(e)
-        result.e2e = time.perf_counter() - st
-
-    if pbar:
-        pbar.update(1)
-    return result
-
-
-async def run_benchmark(
-    host: str,
-    port: int,
-    num_prompts: int,
-    max_concurrency: int,
-    num_warmups: int = 3,
-    voice: str = "vivian",
-    language: str = "English",
-    stream: bool = True,
-) -> BenchmarkResult:
-    api_url = f"http://{host}:{port}/v1/audio/speech"
-
-    connector = aiohttp.TCPConnector(limit=max_concurrency, limit_per_host=max_concurrency, keepalive_timeout=60)
-    session = aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=600))
-
-    if num_warmups > 0:
-        print(f"  Warming up with {num_warmups} requests...")
-        warmup_tasks = [
-            send_tts_request(session, api_url, PROMPTS[i % len(PROMPTS)], voice, language, stream)
-            for i in range(num_warmups)
-        ]
-        await asyncio.gather(*warmup_tasks)
-        print("  Warmup done.")
-
-    request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
-
-    print(f"  Running {num_prompts} requests with concurrency={max_concurrency}...")
-    semaphore = asyncio.Semaphore(max_concurrency)
-    pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
-
-    async def limited_request(prompt):
-        async with semaphore:
-            return await send_tts_request(session, api_url, prompt, voice, language, stream, pbar)
-
-    start_time = time.perf_counter()
-    tasks = [asyncio.create_task(limited_request(p)) for p in request_prompts]
-    results: list[RequestResult] = await asyncio.gather(*tasks)
-    duration = time.perf_counter() - start_time
-    pbar.close()
-
-    await session.close()
-
-    successful = [r for r in results if r.success]
-    failed = [r for r in results if not r.success]
-
-    bench = BenchmarkResult(
-        concurrency=max_concurrency,
-        num_prompts=num_prompts,
-        completed=len(successful),
-        failed=len(failed),
-        duration_s=duration,
-    )
-
-    if successful:
-        ttfps = [r.ttfp * 1000 for r in successful]
-        e2es = [r.e2e * 1000 for r in successful]
-        rtfs = [r.rtf for r in successful]
-        audio_durs = [r.audio_duration for r in successful]
-
-        bench.mean_ttfp_ms = float(np.mean(ttfps))
-        bench.median_ttfp_ms = float(np.median(ttfps))
-        bench.std_ttfp_ms = float(np.std(ttfps))
-        bench.p90_ttfp_ms = float(np.percentile(ttfps, 90))
-        bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
-        bench.p99_ttfp_ms = float(np.percentile(ttfps, 99))
-
-        bench.mean_e2e_ms = float(np.mean(e2es))
-        bench.median_e2e_ms = float(np.median(e2es))
-        bench.std_e2e_ms = float(np.std(e2es))
-        bench.p90_e2e_ms = float(np.percentile(e2es, 90))
-        bench.p95_e2e_ms = float(np.percentile(e2es, 95))
-        bench.p99_e2e_ms = float(np.percentile(e2es, 99))
-
-        bench.mean_rtf = float(np.mean(rtfs))
-        bench.median_rtf = float(np.median(rtfs))
-        bench.std_rtf = float(np.std(rtfs))
-
-        bench.mean_audio_duration_s = float(np.mean(audio_durs))
-        bench.total_audio_duration_s = float(np.sum(audio_durs))
-        bench.audio_throughput = bench.total_audio_duration_s / duration
-        bench.request_throughput = len(successful) / duration
-
-        bench.per_request = [
-            {
-                "ttfp_ms": r.ttfp * 1000,
-                "e2e_ms": r.e2e * 1000,
-                "rtf": r.rtf,
-                "audio_duration_s": r.audio_duration,
-                "prompt": r.prompt,
-            }
-            for r in successful
-        ]
-
-    print(f"\n{'=' * 60}")
-    print(f"  Concurrency: {max_concurrency}  |  Completed: {bench.completed}  |  Failed: {bench.failed}")
-    print(f"  Duration: {duration:.2f}s  |  Throughput: {bench.request_throughput:.2f} req/s")
-    print(
-        f"  TTFP (ms):  mean={bench.mean_ttfp_ms:.1f}  median={bench.median_ttfp_ms:.1f}"
-        f"  p90={bench.p90_ttfp_ms:.1f}  p99={bench.p99_ttfp_ms:.1f}"
-    )
-    print(
-        f"  E2E (ms):   mean={bench.mean_e2e_ms:.1f}  median={bench.median_e2e_ms:.1f}"
-        f"  p90={bench.p90_e2e_ms:.1f}  p99={bench.p99_e2e_ms:.1f}"
-    )
-    print(f"  RTF:        mean={bench.mean_rtf:.3f}  median={bench.median_rtf:.3f}")
-    print(f"  Throughput: {bench.audio_throughput:.2f} audio-sec/wall-sec")
-    print(f"{'=' * 60}\n")
-
-    if failed:
-        for r in failed[:3]:
-            print(f"  [ERROR] {r.error[:200]}")
-
-    return bench
-
-
-async def main(args):
-    all_results = []
-
-    for concurrency in args.max_concurrency:
-        result = await run_benchmark(
-            host=args.host,
-            port=args.port,
-            num_prompts=args.num_prompts,
-            max_concurrency=concurrency,
-            num_warmups=args.num_warmups,
-            voice=args.voice,
-            language=args.language,
-            stream=args.stream,
-        )
-        result.config_name = args.config_name
-        all_results.append(asdict(result))
-
-    result_dir = Path(args.result_dir)
-    result_dir.mkdir(parents=True, exist_ok=True)
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
-
-    with open(result_file, "w") as f:
-        json.dump(all_results, f, indent=2)
-    print(f"Results saved to {result_file}")
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Qwen3-TTS async_chunk benchmark client")
-    parser.add_argument("--host", type=str, default="127.0.0.1")
-    parser.add_argument("--port", type=int, default=8000)
-    parser.add_argument("--num-prompts", type=int, default=50)
-    parser.add_argument("--max-concurrency", type=int, nargs="+", default=[1, 10])
-    parser.add_argument("--num-warmups", type=int, default=3)
-    parser.add_argument("--voice", type=str, default="vivian")
-    parser.add_argument("--language", type=str, default="English")
-    parser.add_argument("--stream", action="store_true", default=True)
-    parser.add_argument("--no-stream", dest="stream", action="store_false")
-    parser.add_argument("--config-name", type=str, default="async_chunk_on")
-    parser.add_argument("--result-dir", type=str, default="results")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    asyncio.run(main(args))

benchmarks/qwen3-tts/vllm_omni/bench_tts_serve.py

@@ -1,371 +0,0 @@
-"""Benchmark client for Qwen3-TTS via /v1/audio/speech endpoint.
-
-Measures TTFP (Time-to-First-Packet), E2E latency, and RTF (Real-Time Factor)
-across configurable concurrency levels. Saves results as JSON for plotting.
-
-Usage:
-    python bench_tts_serve.py \
-        --host 127.0.0.1 --port 8000 \
-        --num-prompts 50 \
-        --max-concurrency 1 4 10 \
-        --result-dir results/
-"""
-
-import argparse
-import asyncio
-import json
-import time
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from pathlib import Path
-
-import aiohttp
-import numpy as np
-from tqdm.asyncio import tqdm
-
-PROMPTS = [
-    "Hello, welcome to the voice synthesis benchmark test.",
-    "She said she would be here by noon, but nobody showed up.",
-    "The quick brown fox jumps over the lazy dog near the riverbank.",
-    "I can't believe how beautiful the sunset looks from up here on the mountain.",
-    "Please remember to bring your identification documents to the appointment tomorrow morning.",
-    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
-    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
-    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
-    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
-    "The train leaves at half past seven, so we need to arrive at the station before then.",
-    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
-    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
-]
-REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
-
-
-@dataclass
-class RequestResult:
-    success: bool = False
-    ttfp: float = 0.0  # Time to first audio packet (seconds)
-    e2e: float = 0.0  # End-to-end latency (seconds)
-    audio_bytes: int = 0  # Total audio bytes received
-    audio_duration: float = 0.0  # Audio duration in seconds (estimated from PCM)
-    rtf: float = 0.0  # Real-time factor = e2e / audio_duration
-    prompt: str = ""
-    error: str = ""
-
-
-@dataclass
-class BenchmarkResult:
-    config_name: str = ""
-    concurrency: int = 0
-    num_prompts: int = 0
-    completed: int = 0
-    failed: int = 0
-    duration_s: float = 0.0
-    # TTFP stats (ms)
-    mean_ttfp_ms: float = 0.0
-    median_ttfp_ms: float = 0.0
-    std_ttfp_ms: float = 0.0
-    p90_ttfp_ms: float = 0.0
-    p95_ttfp_ms: float = 0.0
-    p99_ttfp_ms: float = 0.0
-    # E2E stats (ms)
-    mean_e2e_ms: float = 0.0
-    median_e2e_ms: float = 0.0
-    std_e2e_ms: float = 0.0
-    p90_e2e_ms: float = 0.0
-    p95_e2e_ms: float = 0.0
-    p99_e2e_ms: float = 0.0
-    # RTF stats
-    mean_rtf: float = 0.0
-    median_rtf: float = 0.0
-    std_rtf: float = 0.0
-    p99_rtf: float = 0.0
-    # Audio stats
-    mean_audio_duration_s: float = 0.0
-    total_audio_duration_s: float = 0.0
-    audio_throughput: float = 0.0  # audio_duration / wall_time
-    request_throughput: float = 0.0  # requests / second
-    # Per-request details
-    per_request: list = field(default_factory=list)
-
-
-def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = 24000, sample_width: int = 2) -> float:
-    """Convert raw PCM byte count to duration in seconds."""
-    num_samples = num_bytes / sample_width
-    return num_samples / sample_rate
-
-
-def create_payload(
-    prompt: str, task_type: str = "CustomVoice", voice: str = "vivian", language: str = "English"
-) -> dict:
-    payload = {
-        "input": prompt,
-        "language": language,
-        "stream": True,
-        "response_format": "pcm",
-        "task_type": task_type,
-    }
-
-    if task_type == "Base":
-        payload["ref_audio"] = REF_AUDIO
-        payload["ref_text"] = REF_TEXT
-    elif task_type == "CustomVoice":
-        payload["voice"] = voice
-    elif task_type == "VoiceDesign":
-        payload["instructions"] = INSTRUCT
-
-    return payload
-
-
-async def send_tts_request(
-    session: aiohttp.ClientSession,
-    api_url: str,
-    prompt: str,
-    task_type: str = "CustomVoice",
-    voice: str = "vivian",
-    language: str = "English",
-    pbar: tqdm | None = None,
-) -> RequestResult:
-    """Send a streaming TTS request and measure latency metrics."""
-    payload = create_payload(prompt, task_type, voice, language)
-
-    result = RequestResult(prompt=prompt)
-    st = time.perf_counter()
-
-    try:
-        async with session.post(api_url, json=payload) as response:
-            if response.status != 200:
-                result.error = f"HTTP {response.status}: {await response.text()}"
-                result.success = False
-                return result
-
-            first_chunk = True
-            total_bytes = 0
-
-            async for chunk in response.content.iter_any():
-                if first_chunk and len(chunk) > 0:
-                    result.ttfp = time.perf_counter() - st
-                    first_chunk = False
-                total_bytes += len(chunk)
-
-            result.e2e = time.perf_counter() - st
-            result.audio_bytes = total_bytes
-            result.audio_duration = pcm_bytes_to_duration(total_bytes)
-
-            if result.audio_duration > 0:
-                result.rtf = result.e2e / result.audio_duration
-            result.success = True
-
-    except Exception as e:
-        result.error = str(e)
-        result.success = False
-        result.e2e = time.perf_counter() - st
-
-    if pbar:
-        pbar.update(1)
-    return result
-
-
-async def run_benchmark(
-    host: str,
-    port: int,
-    num_prompts: int,
-    max_concurrency: int,
-    num_warmups: int = 3,
-    task_type: str = "CustomVoice",
-    voice: str = "vivian",
-    language: str = "English",
-) -> BenchmarkResult:
-    """Run benchmark at a given concurrency level."""
-    api_url = f"http://{host}:{port}/v1/audio/speech"
-
-    connector = aiohttp.TCPConnector(
-        limit=max_concurrency,
-        limit_per_host=max_concurrency,
-        keepalive_timeout=60,
-    )
-    session = aiohttp.ClientSession(
-        connector=connector,
-        timeout=aiohttp.ClientTimeout(total=600),
-    )
-
-    # Warmup
-    if num_warmups > 0:
-        print(f"  Warming up with {num_warmups} requests...")
-        warmup_tasks = []
-        for i in range(num_warmups):
-            prompt = PROMPTS[i % len(PROMPTS)]
-            warmup_tasks.append(send_tts_request(session, api_url, prompt, task_type, voice, language))
-        await asyncio.gather(*warmup_tasks)
-        print("  Warmup done.")
-
-    # Build request list
-    request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
-
-    # Run benchmark
-    print(f"  Running {num_prompts} requests with concurrency={max_concurrency}...")
-    semaphore = asyncio.Semaphore(max_concurrency)
-    pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
-
-    async def limited_request(prompt):
-        async with semaphore:
-            return await send_tts_request(session, api_url, prompt, task_type, voice, language, pbar)
-
-    start_time = time.perf_counter()
-    tasks = [asyncio.create_task(limited_request(p)) for p in request_prompts]
-    results: list[RequestResult] = await asyncio.gather(*tasks)
-    duration = time.perf_counter() - start_time
-    pbar.close()
-
-    await session.close()
-
-    # Compute stats
-    successful = [r for r in results if r.success]
-    failed = [r for r in results if not r.success]
-
-    bench = BenchmarkResult(
-        concurrency=max_concurrency,
-        num_prompts=num_prompts,
-        completed=len(successful),
-        failed=len(failed),
-        duration_s=duration,
-    )
-
-    if successful:
-        ttfps = [r.ttfp * 1000 for r in successful]  # convert to ms
-        e2es = [r.e2e * 1000 for r in successful]
-        rtfs = [r.rtf for r in successful]
-        audio_durs = [r.audio_duration for r in successful]
-
-        bench.mean_ttfp_ms = float(np.mean(ttfps))
-        bench.median_ttfp_ms = float(np.median(ttfps))
-        bench.std_ttfp_ms = float(np.std(ttfps))
-        bench.p90_ttfp_ms = float(np.percentile(ttfps, 90))
-        bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
-        bench.p99_ttfp_ms = float(np.percentile(ttfps, 99))
-
-        bench.mean_e2e_ms = float(np.mean(e2es))
-        bench.median_e2e_ms = float(np.median(e2es))
-        bench.std_e2e_ms = float(np.std(e2es))
-        bench.p90_e2e_ms = float(np.percentile(e2es, 90))
-        bench.p95_e2e_ms = float(np.percentile(e2es, 95))
-        bench.p99_e2e_ms = float(np.percentile(e2es, 99))
-
-        bench.mean_rtf = float(np.mean(rtfs))
-        bench.median_rtf = float(np.median(rtfs))
-        bench.std_rtf = float(np.std(rtfs))
-        bench.p99_rtf = float(np.percentile(rtfs, 99))
-
-        bench.mean_audio_duration_s = float(np.mean(audio_durs))
-        bench.total_audio_duration_s = float(np.sum(audio_durs))
-        bench.audio_throughput = bench.total_audio_duration_s / duration
-        bench.request_throughput = len(successful) / duration
-
-        bench.per_request = [
-            {
-                "ttfp_ms": r.ttfp * 1000,
-                "e2e_ms": r.e2e * 1000,
-                "rtf": r.rtf,
-                "audio_duration_s": r.audio_duration,
-                "prompt": r.prompt,
-            }
-            for r in successful
-        ]
-
-    # Print summary in standardized performance template
-    W = 50
-    print("")
-    print(f"{'=' * W}")
-    print(f"{'Serving Benchmark Result':^{W}}")
-    print(f"{'=' * W}")
-    print(f"{'Successful requests:':<40}{bench.completed:<10}")
-    print(f"{'Failed requests:':<40}{bench.failed:<10}")
-    print(f"{'Maximum request concurrency:':<40}{max_concurrency:<10}")
-    print(f"{'Benchmark duration (s):':<40}{duration:<10.2f}")
-    print(f"{'Request throughput (req/s):':<40}{bench.request_throughput:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'End-to-end Latency':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean E2EL (ms):':<40}{bench.mean_e2e_ms:<10.2f}")
-    print(f"{'Median E2EL (ms):':<40}{bench.median_e2e_ms:<10.2f}")
-    print(f"{'P99 E2EL (ms):':<40}{bench.p99_e2e_ms:<10.2f}")
-    print(f"{'=' * W}")
-    print(f"{'Audio Result':^{W}}")
-    print(f"{'=' * W}")
-    print(f"{'Total audio duration generated (s):':<40}{bench.total_audio_duration_s:<10.2f}")
-    print(f"{'Audio throughput (audio duration/s):':<40}{bench.audio_throughput:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'Time to First Packet':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean AUDIO_TTFP (ms):':<40}{bench.mean_ttfp_ms:<10.2f}")
-    print(f"{'Median AUDIO_TTFP (ms):':<40}{bench.median_ttfp_ms:<10.2f}")
-    print(f"{'P99 AUDIO_TTFP (ms):':<40}{bench.p99_ttfp_ms:<10.2f}")
-    print(f"{'-' * W}")
-    print(f"{'Real Time Factor':^{W}}")
-    print(f"{'-' * W}")
-    print(f"{'Mean AUDIO_RTF:':<40}{bench.mean_rtf:<10.3f}")
-    print(f"{'Median AUDIO_RTF:':<40}{bench.median_rtf:<10.3f}")
-    print(f"{'P99 AUDIO_RTF:':<40}{bench.p99_rtf:<10.3f}")
-    print(f"{'=' * W}")
-    print("")
-
-    if failed:
-        for r in failed[:3]:
-            print(f"  [ERROR] {r.error[:200]}")
-
-    return bench
-
-
-async def main(args):
-    all_results = []
-
-    for concurrency in args.max_concurrency:
-        result = await run_benchmark(
-            host=args.host,
-            port=args.port,
-            num_prompts=args.num_prompts,
-            max_concurrency=concurrency,
-            num_warmups=args.num_warmups,
-            task_type=args.task_type,
-            voice=args.voice,
-            language=args.language,
-        )
-        result.config_name = args.config_name
-        all_results.append(asdict(result))
-
-    # Save results
-    result_dir = Path(args.result_dir)
-    result_dir.mkdir(parents=True, exist_ok=True)
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
-
-    with open(result_file, "w") as f:
-        json.dump(all_results, f, indent=2)
-    print(f"Results saved to {result_file}")
-
-    return all_results
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Qwen3-TTS Benchmark Client")
-    parser.add_argument("--host", type=str, default="127.0.0.1")
-    parser.add_argument("--port", type=int, default=8000)
-    parser.add_argument("--num-prompts", type=int, default=50, help="Number of prompts per concurrency level")
-    parser.add_argument(  # noqa: E501
-        "--max-concurrency", type=int, nargs="+", default=[1, 4, 10], help="Concurrency levels to test"
-    )
-    parser.add_argument("--num-warmups", type=int, default=3)
-    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
-    parser.add_argument("--voice", type=str, default="vivian")
-    parser.add_argument("--language", type=str, default="English")
-    parser.add_argument(
-        "--config-name", type=str, default="async_chunk", help="Label for this config (used in filenames)"
-    )
-    parser.add_argument("--result-dir", type=str, default="results")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    asyncio.run(main(args))

benchmarks/qwen3-tts/vllm_omni/plot_async_chunk.py

@@ -1,249 +0,0 @@
-"""Plot TTFP comparison: async_chunk off vs on.
-
-Generates a bar chart with improvement arrows, matching the Qwen3-Omni
-async_chunk benchmark figure style.
-
-Usage:
-    python plot_async_chunk.py \
-        --off results/bench_async_chunk_off_*.json \
-        --on results/bench_async_chunk_on_*.json \
-        --output results/qwen3_tts_async_chunk_ttfp.png
-
-    # Also supports E2E and RTF metrics:
-    python plot_async_chunk.py \
-        --off results/bench_async_chunk_off_*.json \
-        --on results/bench_async_chunk_on_*.json \
-        --metric e2e \
-        --output results/qwen3_tts_async_chunk_e2e.png
-"""
-
-import argparse
-import json
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import numpy as np
-
-METRIC_CONFIG = {
-    "ttfp": {
-        "key": "mean_ttfp_ms",
-        "ylabel": "TTFP (s)",
-        "title": "TTFP (Time to First Audio Packet) - Qwen3-TTS, by concurrency",
-        "to_seconds": True,
-    },
-    "e2e": {
-        "key": "mean_e2e_ms",
-        "ylabel": "E2E (s)",
-        "title": "E2E Latency - Qwen3-TTS, by concurrency",
-        "to_seconds": True,
-    },
-    "rtf": {
-        "key": "mean_rtf",
-        "ylabel": "RTF",
-        "title": "Real-Time Factor - Qwen3-TTS, by concurrency",
-        "to_seconds": False,
-    },
-}
-
-
-def load_results(path: str) -> list[dict]:
-    with open(path) as f:
-        return json.load(f)
-
-
-def plot_ttfp_comparison(
-    off_results: list[dict],
-    on_results: list[dict],
-    metric: str,
-    output_path: str,
-    title_override: str | None = None,
-):
-    cfg = METRIC_CONFIG[metric]
-    key = cfg["key"]
-    to_seconds = cfg["to_seconds"]
-
-    off_map = {r["concurrency"]: r for r in off_results}
-    on_map = {r["concurrency"]: r for r in on_results}
-    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
-
-    off_vals = []
-    on_vals = []
-    for c in concurrencies:
-        v_off = off_map[c][key]
-        v_on = on_map[c][key]
-        if to_seconds:
-            v_off /= 1000.0
-            v_on /= 1000.0
-        off_vals.append(v_off)
-        on_vals.append(v_on)
-
-    fig, ax = plt.subplots(figsize=(8, 6))
-
-    x = np.arange(len(concurrencies))
-    width = 0.3
-
-    ax.bar(x - width / 2, off_vals, width, label="async_chunk off", color="#87CEEB", edgecolor="none")
-    ax.bar(x + width / 2, on_vals, width, label="async_chunk on", color="#FFF8DC", edgecolor="#DDD8B8")
-
-    # Draw improvement arrows and labels
-    for i in range(len(concurrencies)):
-        v_off = off_vals[i]
-        v_on = on_vals[i]
-        if v_on > 0:
-            improvement = v_off / v_on
-        else:
-            improvement = float("inf")
-
-        # Arrow from top of off-bar to top of on-bar
-        arrow_start_x = x[i] - width / 2
-        arrow_start_y = v_off * 0.95
-        arrow_end_x = x[i] + width / 2
-        arrow_end_y = v_on * 1.05
-
-        ax.annotate(
-            "",
-            xy=(arrow_end_x, arrow_end_y),
-            xytext=(arrow_start_x, arrow_start_y),
-            arrowprops=dict(arrowstyle="->", color="red", lw=1.5),
-        )
-
-        # Improvement label
-        label_x = (arrow_start_x + arrow_end_x) / 2
-        label_y = arrow_start_y + (v_off - v_on) * 0.15
-        ax.text(
-            label_x,
-            label_y,
-            f"{improvement:.1f}x improvement",
-            ha="center",
-            va="bottom",
-            fontsize=10,
-            color="red",
-            fontweight="bold",
-        )
-
-    title = title_override or cfg["title"]
-    ax.set_title(title, fontsize=13, fontweight="bold")
-    ax.set_ylabel(cfg["ylabel"], fontsize=12)
-    ax.set_xlabel("Max concurrency", fontsize=12)
-    ax.set_xticks(x)
-    ax.set_xticklabels([str(c) for c in concurrencies])
-    ax.set_yscale("log")
-    ax.legend(loc="upper left", fontsize=11)
-    ax.grid(axis="y", alpha=0.3, linestyle="--")
-    ax.set_axisbelow(True)
-
-    plt.tight_layout()
-    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.savefig(output_path, dpi=150, bbox_inches="tight")
-    print(f"Plot saved to {output_path}")
-    plt.close()
-
-
-def plot_all_metrics(off_results: list[dict], on_results: list[dict], output_path: str):
-    """Generate a 1x3 subplot with TTFP, E2E, and RTF comparisons."""
-    off_map = {r["concurrency"]: r for r in off_results}
-    on_map = {r["concurrency"]: r for r in on_results}
-    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
-
-    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
-    fig.suptitle("Qwen3-TTS: async_chunk on vs off", fontsize=15, fontweight="bold")
-
-    for ax, metric in zip(axes, ["ttfp", "e2e", "rtf"]):
-        cfg = METRIC_CONFIG[metric]
-        key = cfg["key"]
-        to_seconds = cfg["to_seconds"]
-
-        off_vals = []
-        on_vals = []
-        for c in concurrencies:
-            v_off = off_map[c][key]
-            v_on = on_map[c][key]
-            if to_seconds:
-                v_off /= 1000.0
-                v_on /= 1000.0
-            off_vals.append(v_off)
-            on_vals.append(v_on)
-
-        x = np.arange(len(concurrencies))
-        width = 0.3
-        ax.bar(x - width / 2, off_vals, width, label="async_chunk off", color="#87CEEB")
-        ax.bar(x + width / 2, on_vals, width, label="async_chunk on", color="#FFF8DC", edgecolor="#DDD8B8")
-
-        for i in range(len(concurrencies)):
-            if on_vals[i] > 0:
-                improvement = off_vals[i] / on_vals[i]
-                ax.annotate(
-                    "",
-                    xy=(x[i] + width / 2, on_vals[i] * 1.05),
-                    xytext=(x[i] - width / 2, off_vals[i] * 0.95),
-                    arrowprops=dict(arrowstyle="->", color="red", lw=1.5),
-                )
-                label_y = off_vals[i] * 0.85
-                ax.text(x[i], label_y, f"{improvement:.1f}x", ha="center", fontsize=10, color="red", fontweight="bold")
-
-        ax.set_title(cfg["title"].split(" - ")[0], fontsize=12, fontweight="bold")
-        ax.set_ylabel(cfg["ylabel"], fontsize=11)
-        ax.set_xlabel("Max concurrency", fontsize=11)
-        ax.set_xticks(x)
-        ax.set_xticklabels([str(c) for c in concurrencies])
-        if metric != "rtf":
-            ax.set_yscale("log")
-        ax.legend(fontsize=9)
-        ax.grid(axis="y", alpha=0.3, linestyle="--")
-        ax.set_axisbelow(True)
-
-    plt.tight_layout()
-    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.savefig(output_path, dpi=150, bbox_inches="tight")
-    print(f"Plot saved to {output_path}")
-    plt.close()
-
-
-def print_table(off_results: list[dict], on_results: list[dict]):
-    off_map = {r["concurrency"]: r for r in off_results}
-    on_map = {r["concurrency"]: r for r in on_results}
-    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
-
-    print("\n## Benchmark Results: async_chunk off vs on\n")
-    print("| Metric | Concurrency | async_chunk off | async_chunk on | Improvement |")
-    print("| --- | --- | --- | --- | --- |")
-
-    for name, key, fmt in [
-        ("TTFP (ms)", "mean_ttfp_ms", ".1f"),
-        ("E2E (ms)", "mean_e2e_ms", ".1f"),
-        ("RTF", "mean_rtf", ".3f"),
-        ("Throughput", "audio_throughput", ".2f"),
-    ]:
-        for c in concurrencies:
-            v_off = off_map[c].get(key, 0)
-            v_on = on_map[c].get(key, 0)
-            if v_on > 0 and key != "audio_throughput":
-                ratio = f"{v_off / v_on:.1f}x"
-            elif v_off > 0 and key == "audio_throughput":
-                ratio = f"{v_on / v_off:.1f}x"
-            else:
-                ratio = "N/A"
-            print(f"| {name} | {c} | {v_off:{fmt}} | {v_on:{fmt}} | {ratio} |")
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Plot async_chunk comparison for Qwen3-TTS")
-    parser.add_argument("--off", type=str, required=True, help="JSON results for async_chunk off")
-    parser.add_argument("--on", type=str, required=True, help="JSON results for async_chunk on")
-    parser.add_argument("--metric", type=str, default="ttfp", choices=["ttfp", "e2e", "rtf", "all"])
-    parser.add_argument("--output", type=str, default="results/qwen3_tts_async_chunk.png")
-    parser.add_argument("--title", type=str, default=None, help="Custom title override")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    off_results = load_results(args.off)
-    on_results = load_results(args.on)
-
-    print_table(off_results, on_results)
-
-    if args.metric == "all":
-        plot_all_metrics(off_results, on_results, args.output)
-    else:
-        plot_ttfp_comparison(off_results, on_results, args.metric, args.output, args.title)

benchmarks/qwen3-tts/vllm_omni/run_async_chunk_benchmark.sh

@@ -1,169 +0,0 @@
-#!/bin/bash
-# Qwen3-TTS async_chunk on vs off Benchmark
-#
-# Starts two servers (async_chunk on and off), benchmarks both,
-# and generates comparison plots.
-#
-# Usage:
-#   bash run_async_chunk_benchmark.sh
-#
-# Environment variables:
-#   GPU_DEVICE       - GPU index (default: 0)
-#   MODEL            - Model path (default: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice)
-#   NUM_PROMPTS      - Prompts per concurrency level (default: 50)
-#   CONCURRENCY      - Space-separated concurrency levels (default: "1 10")
-#   PORT_ON          - Port for async_chunk on server (default: 8000)
-#   PORT_OFF         - Port for async_chunk off server (default: 8001)
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
-cd "$PROJECT_ROOT"
-
-GPU_DEVICE="${GPU_DEVICE:-0}"
-MODEL="${MODEL:-Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice}"
-NUM_PROMPTS="${NUM_PROMPTS:-50}"
-CONCURRENCY="${CONCURRENCY:-1 10}"
-NUM_WARMUPS="${NUM_WARMUPS:-3}"
-PORT_ON="${PORT_ON:-8000}"
-PORT_OFF="${PORT_OFF:-8001}"
-RESULT_DIR="${SCRIPT_DIR}/results"
-TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
-
-# The bundled ``vllm_omni/deploy/qwen3_tts.yaml`` is auto-loaded by the model
-# registry; no ``--deploy-config`` flag needed on the default (ON) path.
-# async_chunk OFF is selected by the ``--no-async-chunk`` CLI flag —
-# the single ``qwen3_tts`` pipeline dispatches to the end-to-end codec
-# processor when ``deploy.async_chunk`` is false.
-
-mkdir -p "${RESULT_DIR}"
-
-echo "============================================================"
-echo " Qwen3-TTS async_chunk Benchmark"
-echo "============================================================"
-echo " GPU:            ${GPU_DEVICE}"
-echo " Model:          ${MODEL}"
-echo " Prompts:        ${NUM_PROMPTS}"
-echo " Concurrency:    ${CONCURRENCY}"
-echo " Port (on/off):  ${PORT_ON} / ${PORT_OFF}"
-echo " Results:        ${RESULT_DIR}"
-echo "============================================================"
-
-cleanup() {
-    echo "Cleaning up servers..."
-    kill "$PID_ON" 2>/dev/null || true
-    kill "$PID_OFF" 2>/dev/null || true
-    wait "$PID_ON" 2>/dev/null || true
-    wait "$PID_OFF" 2>/dev/null || true
-}
-trap cleanup EXIT
-
-wait_for_server() {
-    local port=$1
-    local name=$2
-    local max_wait=300
-    local elapsed=0
-    echo "Waiting for ${name} server on port ${port}..."
-    while ! curl -s "http://localhost:${port}/health" >/dev/null 2>&1; do
-        sleep 5
-        elapsed=$((elapsed + 5))
-        if [ $elapsed -ge $max_wait ]; then
-            echo "ERROR: ${name} server failed to start within ${max_wait}s"
-            exit 1
-        fi
-    done
-    echo "${name} server ready (${elapsed}s)"
-}
-
-# ---- Phase 1: Start async_chunk ON server ----
-echo ""
-echo "[Phase 1] Starting async_chunk ON server on port ${PORT_ON}..."
-CUDA_VISIBLE_DEVICES=${GPU_DEVICE} vllm-omni serve "${MODEL}" \
-    --host 0.0.0.0 --port "${PORT_ON}" \
-    --trust-remote-code --enforce-eager --omni \
-    > "${RESULT_DIR}/server_on_${TIMESTAMP}.log" 2>&1 &
-PID_ON=$!
-
-wait_for_server "${PORT_ON}" "async_chunk_on"
-
-echo "[Phase 1] Benchmarking async_chunk ON..."
-# shellcheck disable=SC2086
-python "${SCRIPT_DIR}/bench_async_chunk.py" \
-    --host 127.0.0.1 --port "${PORT_ON}" \
-    --config-name "async_chunk_on" \
-    --num-prompts "${NUM_PROMPTS}" \
-    --max-concurrency ${CONCURRENCY} \
-    --num-warmups "${NUM_WARMUPS}" \
-    --result-dir "${RESULT_DIR}"
-
-echo "[Phase 1] Stopping async_chunk ON server..."
-kill "$PID_ON" 2>/dev/null || true
-wait "$PID_ON" 2>/dev/null || true
-sleep 5
-
-# ---- Phase 2: Start async_chunk OFF server ----
-echo ""
-echo "[Phase 2] Starting async_chunk OFF server on port ${PORT_OFF}..."
-CUDA_VISIBLE_DEVICES=${GPU_DEVICE} vllm-omni serve "${MODEL}" \
-    --no-async-chunk \
-    --host 0.0.0.0 --port "${PORT_OFF}" \
-    --trust-remote-code --enforce-eager --omni \
-    > "${RESULT_DIR}/server_off_${TIMESTAMP}.log" 2>&1 &
-PID_OFF=$!
-
-wait_for_server "${PORT_OFF}" "async_chunk_off"
-
-echo "[Phase 2] Benchmarking async_chunk OFF (non-streaming)..."
-# shellcheck disable=SC2086
-python "${SCRIPT_DIR}/bench_async_chunk.py" \
-    --host 127.0.0.1 --port "${PORT_OFF}" \
-    --config-name "async_chunk_off" \
-    --num-prompts "${NUM_PROMPTS}" \
-    --max-concurrency ${CONCURRENCY} \
-    --num-warmups "${NUM_WARMUPS}" \
-    --no-stream \
-    --result-dir "${RESULT_DIR}"
-
-echo "[Phase 2] Stopping async_chunk OFF server..."
-kill "$PID_OFF" 2>/dev/null || true
-wait "$PID_OFF" 2>/dev/null || true
-
-# ---- Phase 3: Plot results ----
-echo ""
-echo "[Phase 3] Generating plots..."
-
-# Find the latest result files
-RESULT_ON=$(ls -t "${RESULT_DIR}"/bench_async_chunk_on_*.json 2>/dev/null | head -1)
-RESULT_OFF=$(ls -t "${RESULT_DIR}"/bench_async_chunk_off_*.json 2>/dev/null | head -1)
-
-if [ -z "$RESULT_ON" ] || [ -z "$RESULT_OFF" ]; then
-    echo "ERROR: Could not find result files. Check logs in ${RESULT_DIR}/"
-    exit 1
-fi
-
-echo "  ON results:  ${RESULT_ON}"
-echo "  OFF results: ${RESULT_OFF}"
-
-# TTFP comparison (main figure)
-python "${SCRIPT_DIR}/plot_async_chunk.py" \
-    --off "${RESULT_OFF}" \
-    --on "${RESULT_ON}" \
-    --metric ttfp \
-    --output "${RESULT_DIR}/qwen3_tts_async_chunk_ttfp.png"
-
-# All metrics comparison
-python "${SCRIPT_DIR}/plot_async_chunk.py" \
-    --off "${RESULT_OFF}" \
-    --on "${RESULT_ON}" \
-    --metric all \
-    --output "${RESULT_DIR}/qwen3_tts_async_chunk_all.png"
-
-echo ""
-echo "============================================================"
-echo " Benchmark complete!"
-echo " Results: ${RESULT_DIR}/"
-echo " Plots:"
-echo "   - ${RESULT_DIR}/qwen3_tts_async_chunk_ttfp.png"
-echo "   - ${RESULT_DIR}/qwen3_tts_async_chunk_all.png"
-echo "============================================================"

benchmarks/tts/README.md

@@ -0,0 +1,227 @@
+# TTS Universal Benchmark
+
+A model-agnostic serving benchmark for TTS models in vllm-omni. One CLI
+(`bench_tts.py`) + one YAML registry (`model_configs.yaml`) drive perf and
+quality runs for every registered checkpoint: **Qwen3-TTS** (Base / CustomVoice)
+and **VoxCPM2** today, more to come.
+
+The same three task types — `voice_clone`, `default_voice`, `voice_design` —
+are wired into both the manual CLI and the DFX nightly CI matrix
+(`tests/dfx/perf/tests/test_tts.json`).
+
+## Quick start
+
+### 1. Start the server
+
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --port 8000
+```
+
+The server auto-loads its Deploy YAML from `vllm_omni/deploy/qwen3_tts.yaml`
+(Pipeline + Deploy schema introduced in #2383). No `--stage-configs-path` or
+`--deploy-config` flag is needed for any registered model.
+
+### 2. Run the benchmark (`vllm bench serve --omni`)
+
+The primary, directly-controllable path. Copy-paste one of these and tweak
+any bench flag (sampling params, endpoint, extra body, warmups, etc.):
+
+#### voice_clone (Qwen3-TTS-Base, seed-tts dataset)
+
+```bash
+vllm bench serve --omni \
+    --host 127.0.0.1 --port 8000 \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --backend openai-audio-speech \
+    --endpoint /v1/audio/speech \
+    --dataset-name seed-tts \
+    --dataset-path /path/to/seed-tts-eval \
+    --seed-tts-locale en \
+    --num-prompts 20 --num-warmups 2 \
+    --extra-body '{"task_type":"Base"}' \
+    --max-concurrency 1 --request-rate inf \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --save-result --result-dir ./results
+```
+
+#### default_voice (Qwen3-TTS-CustomVoice, bundled seed_tts_smoke)
+
+```bash
+vllm bench serve --omni \
+    --host 127.0.0.1 --port 8000 \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --backend openai-audio-speech \
+    --endpoint /v1/audio/speech \
+    --dataset-name seed-tts-text \
+    --dataset-path benchmarks/build_dataset/seed_tts_smoke \
+    --seed-tts-locale en \
+    --num-prompts 20 --num-warmups 2 \
+    --extra-body '{"voice":"Vivian","language":"English","task_type":"CustomVoice"}' \
+    --max-concurrency 1 --request-rate inf \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --save-result --result-dir ./results
+```
+
+#### voice_design (Qwen3-TTS-CustomVoice, bundled seed_tts_design)
+
+```bash
+vllm bench serve --omni \
+    --host 127.0.0.1 --port 8000 \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --backend openai-audio-speech \
+    --endpoint /v1/audio/speech \
+    --dataset-name seed-tts-design \
+    --dataset-path benchmarks/build_dataset/seed_tts_design \
+    --seed-tts-locale en \
+    --num-prompts 20 --num-warmups 2 \
+    --extra-body '{"task_type":"VoiceDesign","language":"English"}' \
+    --max-concurrency 1 --request-rate inf \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --save-result --result-dir ./results
+```
+
+#### Add WER / SIM / UTMOS to any of the above
+
+Append `--seed-tts-wer-eval` (and optionally `SEED_TTS_EVAL_DEVICE=cuda:0`
+in the env, per PR #2558). This triggers the seed-tts-eval protocol:
+Whisper-large-v3 ASR → WER, WavLM embeddings → SIM, balacoon/utmos → UTMOS.
+
+### 3. Convenience wrapper (`bench_tts.py`)
+
+If you're running the **canonical** configuration for a registered model,
+`bench_tts.py` loads the right defaults from `model_configs.yaml` and
+emits the exact `vllm bench serve --omni` command above — useful for
+concurrency sweeps and multi-task runs:
+
+```bash
+# Smallest smoke — 5 prompts, concurrency=1
+python benchmarks/tts/bench_tts.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task voice_clone \
+    --dataset-path /path/to/seed-tts-eval \
+    --concurrency 1 --num-prompts 5 \
+    --output-dir ./results
+
+# Full concurrency sweep
+python benchmarks/tts/bench_tts.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task voice_clone \
+    --dataset-path /path/to/seed-tts-eval \
+    --concurrency 1 2 4 8 16 32 \
+    --num-prompts 20 \
+    --output-dir ./results
+
+# With WER / SIM / UTMOS quality eval (adds ASR + embedding compute)
+python benchmarks/tts/bench_tts.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task voice_clone \
+    --dataset-path /path/to/seed-tts-eval \
+    --wer-eval \
+    --concurrency 4 --num-prompts 200 \
+    --output-dir ./results
+```
+
+### 4. Plot a sweep
+
+```bash
+python benchmarks/tts/plot_results.py \
+    --results ./results/*.json \
+    --output ./results/curve.png
+```
+
+Outputs TTFP / RTF / throughput curves (and a markdown table) for every
+`(task, concurrency)` combination in the result set.
+
+## Task types
+
+| Task            | Dataset           | Request body                                        | Checkpoints that support it              |
+|-----------------|-------------------|-----------------------------------------------------|------------------------------------------|
+| `voice_clone`   | `seed-tts`        | `ref_audio` + `ref_text` + `task_type=Base`         | `Qwen3-TTS-*-Base`, `VoxCPM2`            |
+| `default_voice` | `seed-tts-text`   | `voice=Vivian` + `task_type=CustomVoice`            | `Qwen3-TTS-*-CustomVoice`                |
+| `voice_design`  | `seed-tts-design` | `instructions=<natural-language description>` + `task_type=VoiceDesign` | `Qwen3-TTS-*-CustomVoice` |
+
+**`-CustomVoice` checkpoints do NOT ship `speaker_encoder` weights**, so
+voice_clone requests raise `ValueError` at model runtime. Use `-Base` for
+voice_clone.
+
+## Adding a new TTS model
+
+Drop an entry into `model_configs.yaml` — no Python changes required:
+
+```yaml
+models:
+  <org>/<model-id>:
+    supported_tasks: [voice_clone]          # or default_voice / voice_design
+    backend: openai-audio-speech            # vllm bench serve backend
+    endpoint: /v1/audio/speech              # OpenAI-compatible endpoint
+    task_extra_body:                        # merged into every request's body
+      voice_clone:
+        task_type: Base
+```
+
+Then add the model's Deploy YAML under `vllm_omni/deploy/<model>.yaml`
+(Pipeline + Deploy schema) and it's immediately benchable.
+
+## Datasets
+
+| Dataset            | Bundled? | Format            | Source                                                         |
+|--------------------|----------|-------------------|----------------------------------------------------------------|
+| `seed-tts-design`  | ✅       | 5-field meta.lst  | `benchmarks/build_dataset/seed_tts_design/en/meta.lst` (20 prompts) |
+| `seed_tts_smoke`   | ✅       | 4-field meta.lst  | `benchmarks/build_dataset/seed_tts_smoke/en/meta.lst` (20 text-only) |
+| `seed-tts`         | ❌       | 4-field meta.lst + WAVs | Google-Drive: [BytedanceSpeech/seed-tts-eval][seedtts] (~1.2 GB) |
+| `seed-tts-text`    | ❌       | 4-field meta.lst  | Same archive as `seed-tts` (wav column unused)                 |
+
+[seedtts]: https://github.com/BytedanceSpeech/seed-tts-eval
+
+For manual voice_clone / default_voice runs against the full corpus, follow
+`benchmarks/build_dataset/download_process_data_seedtts.md` and point
+`--dataset-path` at the extracted `seedtts_testset` directory.
+
+## DFX nightly CI
+
+`tests/dfx/perf/tests/test_tts.json` wires three perf regimes plus quality:
+
+| eval_phase    | concurrency | purpose                                                 | Baseline metrics                        |
+|---------------|-------------|---------------------------------------------------------|-----------------------------------------|
+| `latency`     | 1           | Single-request TTFP / RTF SLO                           | `median_audio_ttfp_ms`, `median_audio_rtf` |
+| `throughput`  | 8           | Codec-batching cliff sentinel (PDF #272 concurrency≥8)  | `median_audio_ttfp_ms`, `median_audio_rtf` |
+| `quality`     | 4           | WER / SIM / UTMOS regression (disabled in CI by default)| `mean_audio_rtf`                        |
+
+Why `median_*` for latency/throughput and `mean_*` for quality: latency
+distributions have cold-start tails that drag the mean; quality aggregates
+over 200 prompts so single-request outliers don't matter.
+
+Quality entries are `enabled: false` in CI because seed-tts-eval is not
+staged in the Buildkite container (matches the precedent in
+PR #2558 — quality runs are manual / release-validation, not nightly).
+
+## Concurrency cliff regression sentinel
+
+Observed on H20-3e, Qwen3-TTS-1.7B (measured pre-merge on this branch):
+
+| Task          | Model         | c=1    | c=4    | **c=8**    | c=16   | c=32   |
+|---------------|---------------|--------|--------|------------|--------|--------|
+| voice_clone   | 1.7B-Base     | RTF 0.15 / TTFP 165ms | 0.28 / 412ms | **0.49 / 1701ms** | 0.72 / 3355ms | 0.77 / 3772ms |
+| voice_design  | 1.7B-CustomVoice | RTF 0.08 / TTFP 53ms  | 0.11 / 154ms | **0.21 / 872ms**  | 0.33 / 1801ms | 0.38 / 1989ms |
+
+Both models show a **4–6× TTFP jump from c=4 to c=8** while audio throughput
+saturates around c=4–8 — the codec-bs=1 bottleneck documented in
+vllm-project/vllm-omni#272. The `throughput` CI regime at c=8 is the
+sentinel for regressions in this area.
+
+## File layout
+
+```
+benchmarks/tts/
+├── README.md                  (this file)
+├── bench_tts.py               CLI — serve-mode benchmark driver
+├── bench_voxcpm_offline.py    CLI — offline VoxCPM benchmark (sync + streaming)
+├── plot_results.py            Generate per-task / per-concurrency curves
+└── model_configs.yaml         Model registry (supported tasks + extra body)
+```
+
+## Related
+
+- Upstream seed-tts-eval integration: vllm-project/vllm-omni#2558
+- Pipeline + Deploy schema: vllm-project/vllm-omni#2383
+- Concurrency cliff RFC: vllm-project/vllm-omni#272

benchmarks/tts/bench_tts.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""Universal TTS benchmark CLI for vllm-omni.
+
+Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
+``model_configs.yaml``.  Supports Qwen3-TTS, VoxCPM2, and any future TTS
+model registered in the config file -- no code changes needed to add models.
+
+Usage::
+
+    python benchmarks/tts/bench_tts.py \\
+        --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \\
+        --task voice_clone \\
+        --locale en \\
+        --concurrency 1 4 \\
+        --num-prompts 20 \\
+        --dataset-path /path/to/seed-tts-eval \\
+        --host localhost --port 8000
+
+See ``--help`` for full option list.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _vllm_omni_bin() -> str:
+    """Return the vllm-omni (or vllm) binary co-located with the current Python."""
+    bin_dir = Path(sys.executable).parent
+    for candidate in ("vllm-omni", "vllm"):
+        p = bin_dir / candidate
+        if p.is_file():
+            return str(p)
+    return "vllm-omni"  # fall back and let the shell resolve it
+
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+_SCRIPT_DIR = Path(__file__).resolve().parent
+_DEFAULT_MODEL_CONFIGS = _SCRIPT_DIR / "model_configs.yaml"
+
+# Maps task name to the dataset_name used with vllm bench serve
+_TASK_TO_DATASET: dict[str, str] = {
+    "voice_clone": "seed-tts",
+    "default_voice": "seed-tts-text",
+    "voice_design": "seed-tts-design",
+}
+
+# Default design dataset path (bundled with the repo)
+_DEFAULT_DESIGN_DATASET_PATH = str(_REPO_ROOT / "benchmarks" / "build_dataset" / "seed_tts_design")
+
+
+def load_model_configs(path: Path) -> dict[str, Any]:
+    """Load model registry from YAML file."""
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+    return data.get("models", {})
+
+
+def build_bench_args(
+    *,
+    host: str,
+    port: int,
+    model: str,
+    task: str,
+    model_cfg: dict[str, Any],
+    locale: str,
+    num_prompts: int,
+    concurrency: int | None,
+    dataset_path: str | None,
+    wer_eval: bool,
+    output_dir: str | None,
+    result_filename: str | None,
+    extra_cli_args: list[str],
+) -> list[str]:
+    """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
+    dataset_name = _TASK_TO_DATASET[task]
+    backend: str = model_cfg["backend"]
+    endpoint: str = model_cfg["endpoint"]
+    task_extra_body: dict[str, Any] = (model_cfg.get("task_extra_body") or {}).get(task) or {}
+
+    # Resolve dataset path
+    if dataset_path:
+        resolved_dataset_path = dataset_path
+    elif task == "voice_design":
+        resolved_dataset_path = _DEFAULT_DESIGN_DATASET_PATH
+    else:
+        resolved_dataset_path = None
+
+    cmd = [
+        _vllm_omni_bin(),
+        "bench",
+        "serve",
+        "--omni",
+        "--host",
+        host,
+        "--port",
+        str(port),
+        "--model",
+        model,
+        "--backend",
+        backend,
+        "--endpoint",
+        endpoint,
+        "--dataset-name",
+        dataset_name,
+        "--num-prompts",
+        str(num_prompts),
+        "--num-warmups",
+        "2",
+        "--percentile-metrics",
+        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+    ]
+
+    if resolved_dataset_path:
+        cmd += ["--dataset-path", resolved_dataset_path]
+
+    if locale:
+        cmd += ["--seed-tts-locale", locale]
+
+    if task_extra_body:
+        cmd += ["--extra-body", json.dumps(task_extra_body, separators=(",", ":"))]
+
+    if concurrency is not None:
+        cmd += ["--max-concurrency", str(concurrency), "--request-rate", "inf"]
+
+    if wer_eval:
+        cmd.append("--seed-tts-wer-eval")
+
+    if output_dir or result_filename:
+        out_dir = output_dir or "."
+        os.makedirs(out_dir, exist_ok=True)
+        cmd += ["--save-result", "--result-dir", out_dir]
+        if result_filename:
+            cmd += ["--result-filename", result_filename]
+
+    cmd += extra_cli_args
+    return cmd
+
+
+def run_one_benchmark(cmd: list[str]) -> dict[str, Any] | None:
+    """Run a single benchmark subprocess and return parsed JSON result if available."""
+    print(f"\n{'=' * 60}")
+    print("Running:", " ".join(cmd))
+    print("=" * 60)
+    result = subprocess.run(cmd, check=False)
+    if result.returncode != 0:
+        print(f"[bench_tts] WARNING: benchmark exited with code {result.returncode}")
+        return None
+    # If --save-result was used, find the result file
+    try:
+        result_dir_idx = cmd.index("--result-dir")
+        result_dir = Path(cmd[result_dir_idx + 1])
+        if "--result-filename" in cmd:
+            fname_idx = cmd.index("--result-filename")
+            result_file = result_dir / cmd[fname_idx + 1]
+        else:
+            # find most recently modified json
+            jsons = sorted(result_dir.glob("result_*.json"), key=lambda p: p.stat().st_mtime)
+            result_file = jsons[-1] if jsons else None
+        if result_file and result_file.is_file():
+            return json.loads(result_file.read_text(encoding="utf-8"))
+    except (ValueError, IndexError, OSError):
+        pass
+    return None
+
+
+def print_summary_table(results: list[dict[str, Any]]) -> None:
+    """Print a unified metrics table across all (task, concurrency) runs."""
+    if not results:
+        return
+    header = (
+        f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
+        f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
+    )
+    print(f"\n{'=' * len(header)}")
+    print("BENCHMARK SUMMARY")
+    print("=" * len(header))
+    print(header)
+    print("-" * len(header))
+    for r in results:
+        task = r.get("_task", "?")
+        conc = r.get("_concurrency", "?")
+        rtf = r.get("mean_audio_rtf", float("nan"))
+        ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
+        throughput = r.get("audio_throughput", float("nan"))
+        wer = r.get("seed_tts_mean_wer", float("nan"))
+        sim = r.get("seed_tts_mean_sim", float("nan"))
+        utmos = r.get("seed_tts_mean_utmos", float("nan"))
+
+        def fmt(v: float, digits: int = 3) -> str:
+            return f"{v:.{digits}f}" if not math.isnan(v) else "  n/a"
+
+        print(
+            f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
+            f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
+        )
+    print("=" * len(header))
+
+
+def main() -> None:
+    """Entry point for the universal TTS benchmark CLI."""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--model", required=True, help="HuggingFace model ID (e.g. Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice)"
+    )
+    parser.add_argument("--task", default="all", help="Task type: voice_clone | default_voice | voice_design | all")
+    parser.add_argument("--locale", default="en", choices=["en", "zh"])
+    parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4], metavar="N")
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        nargs="+",
+        default=[20],
+        metavar="N",
+        help="Number of prompts per run. If one value, applied to all concurrency levels.",
+    )
+    parser.add_argument(
+        "--dataset-path", default=None, help="Root of seed-tts-eval dataset (required for voice_clone/default_voice)"
+    )
+    parser.add_argument("--wer-eval", action="store_true", help="Enable WER/SIM/UTMOS quality eval")
+    parser.add_argument("--output-dir", default=None, help="Directory to save result JSON files")
+    parser.add_argument("--host", default="localhost")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--model-configs", default=str(_DEFAULT_MODEL_CONFIGS), help="Path to model_configs.yaml")
+    parser.add_argument("extra", nargs=argparse.REMAINDER, help="Extra args passed directly to vllm bench serve")
+    args = parser.parse_args()
+
+    model_configs = load_model_configs(Path(args.model_configs))
+    if args.model not in model_configs:
+        known = "\n  ".join(model_configs.keys())
+        print(f"[bench_tts] ERROR: model '{args.model}' not in model_configs.yaml.\nKnown models:\n  {known}")
+        sys.exit(1)
+
+    model_cfg = model_configs[args.model]
+    supported_tasks: list[str] = model_cfg.get("supported_tasks", [])
+
+    tasks_to_run: list[str]
+    if args.task == "all":
+        tasks_to_run = supported_tasks
+    elif args.task in supported_tasks:
+        tasks_to_run = [args.task]
+    else:
+        print(
+            f"[bench_tts] ERROR: task '{args.task}' not supported by {args.model}.\nSupported tasks: {supported_tasks}"
+        )
+        sys.exit(1)
+
+    # Align num_prompts list with concurrency list
+    num_prompts_list: list[int] = args.num_prompts
+    if len(num_prompts_list) == 1:
+        num_prompts_list = num_prompts_list * len(args.concurrency)
+    elif len(num_prompts_list) != len(args.concurrency):
+        print(
+            f"[bench_tts] ERROR: --num-prompts ({len(num_prompts_list)} values) must be "
+            f"length 1 or match --concurrency ({len(args.concurrency)} values)."
+        )
+        sys.exit(1)
+
+    all_results: list[dict[str, Any]] = []
+
+    for task in tasks_to_run:
+        for concurrency, num_prompts in zip(args.concurrency, num_prompts_list):
+            ts = datetime.now().strftime("%Y%m%d-%H%M%S")
+            result_filename = f"bench_tts_{args.model.replace('/', '_')}_{task}_c{concurrency}_{ts}.json"
+            cmd = build_bench_args(
+                host=args.host,
+                port=args.port,
+                model=args.model,
+                task=task,
+                model_cfg=model_cfg,
+                locale=args.locale,
+                num_prompts=num_prompts,
+                concurrency=concurrency,
+                dataset_path=args.dataset_path,
+                wer_eval=args.wer_eval,
+                output_dir=args.output_dir,
+                result_filename=result_filename,
+                extra_cli_args=args.extra or [],
+            )
+            result = run_one_benchmark(cmd)
+            if result is not None:
+                result["_task"] = task
+                result["_concurrency"] = concurrency
+                all_results.append(result)
+                # Persist the metadata so plot_results.py can pick it up.
+                if args.output_dir and result_filename:
+                    result_path = Path(args.output_dir) / result_filename
+                    if result_path.is_file():
+                        result_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
+
+    print_summary_table(all_results)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tts/bench_voxcpm_offline.py

@@ -6,6 +6,27 @@ Supports both:
 - text-only synthesis
 - voice cloning
 - text/clone batch inputs from txt or jsonl
+
+Usage::
+
+    # Sync (default voice)
+    python benchmarks/tts/bench_voxcpm_offline.py \\
+        --model /path/to/VoxCPM \\
+        --text "Hello world" \\
+        --output-dir results/audio/
+
+    # Streaming (async_chunk)
+    python benchmarks/tts/bench_voxcpm_offline.py \\
+        --model /path/to/VoxCPM \\
+        --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \\
+        --txt-prompts prompts.txt \\
+        --output-dir results/audio/
+
+    # Voice cloning batch via JSONL
+    python benchmarks/tts/bench_voxcpm_offline.py \\
+        --model /path/to/VoxCPM \\
+        --jsonl-prompts prompts.jsonl \\
+        --output-dir results/audio/
 """
 
 from __future__ import annotations
@@ -26,7 +47,21 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni, Omni
 
-REPO_ROOT = Path(__file__).resolve().parents[3]
+
+def _find_repo_root(start: Path) -> Path:
+    """Walk up from ``start`` until a repo marker is found.
+
+    Falls back to ``parents[2]`` for backwards compatibility if no marker hits
+    (which can only happen in unusual checkouts — the tree should always have
+    pyproject.toml + vllm_omni/ at the top level).
+    """
+    for candidate in [start, *start.parents]:
+        if (candidate / "pyproject.toml").is_file() and (candidate / "vllm_omni").is_dir():
+            return candidate
+    return start.parents[2]
+
+
+REPO_ROOT = _find_repo_root(Path(__file__).resolve())
 DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
 DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
 
@@ -473,9 +508,6 @@ def parse_args():
 
 def _is_streaming_stage_config(stage_configs_path: str) -> bool:
     cfg_name = Path(stage_configs_path).name.lower()
-    # Keep routing purely config-path based:
-    # - voxcpm.yaml => sync
-    # - voxcpm_async_chunk.yaml => streaming
     return "async_chunk" in cfg_name
 
 

benchmarks/tts/model_configs.yaml

@@ -0,0 +1,39 @@
+# Universal TTS benchmark model registry.
+# Maps HuggingFace model ID → supported tasks + per-task extra body fields.
+# To add a new TTS model: add an entry here. No code changes required.
+#
+# The server auto-loads its Deploy YAML from vllm_omni/deploy/<model>.yaml via
+# the Pipeline + Deploy schema introduced in #2383, so no stage_config path
+# is tracked here.
+
+models:
+  # -CustomVoice checkpoints lack speaker_encoder weights, so voice_clone is
+  # NOT supported (an attempt raises ValueError from _extract_speaker_embedding
+  # at model runtime).  Use -Base for voice_clone.
+  Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice:
+    supported_tasks: [default_voice, voice_design]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    task_extra_body:
+      default_voice:
+        voice: Vivian
+        language: English
+        task_type: CustomVoice
+      voice_design:
+        task_type: VoiceDesign
+        language: English
+
+  Qwen/Qwen3-TTS-12Hz-1.7B-Base:
+    supported_tasks: [voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    task_extra_body:
+      voice_clone:
+        task_type: Base
+
+  openbmb/VoxCPM2:
+    supported_tasks: [voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    task_extra_body:
+      voice_clone: {}

benchmarks/tts/plot_results.py

@@ -0,0 +1,324 @@
+"""Plot universal TTS benchmark results.
+
+Reads JSON files saved by ``bench_tts.py`` (via ``vllm bench serve --omni``)
+and generates comparison bar charts grouped by task type.
+
+Metrics plotted:
+- AUDIO_TTFP  (mean audio time-to-first-packet, ms)
+- E2EL        (mean end-to-end latency, ms)
+- Audio RTF   (mean real-time factor)
+- Audio throughput (audio-seconds / wall-second)
+
+Quality metrics (WER / SIM / UTMOS) are printed in a table when present.
+
+Usage::
+
+    # Single run — one JSON per task, all in results/
+    python benchmarks/tts/plot_results.py \\
+        --results results/bench_tts_*.json \\
+        --output results/tts_benchmark.png
+
+    # Compare two runs (e.g. async_chunk on vs off)
+    python benchmarks/tts/plot_results.py \\
+        --results run_a/bench_tts_*.json \\
+        --results run_b/bench_tts_*.json \\
+        --labels "async_chunk_on" "async_chunk_off" \\
+        --output results/comparison.png
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+# ---------------------------------------------------------------------------
+# JSON loading
+# ---------------------------------------------------------------------------
+
+
+def load_run(paths: list[str]) -> list[dict]:
+    """Load and merge all JSON files for one run into a flat list of records.
+
+    Each record is expected to have at least ``_concurrency`` (int) and
+    ``_task`` (str) keys injected by ``bench_tts.py``.  Records that come
+    from a file that contains a list are flattened.
+    """
+    records: list[dict] = []
+    for p in paths:
+        raw = json.loads(Path(p).read_text(encoding="utf-8"))
+        if isinstance(raw, list):
+            records.extend(raw)
+        elif isinstance(raw, dict):
+            records.append(raw)
+    return records
+
+
+def _get(record: dict, key: str) -> float:
+    v = record.get(key, float("nan"))
+    if v is None or (isinstance(v, float) and math.isnan(v)):
+        return float("nan")
+    try:
+        return float(v)
+    except (TypeError, ValueError):
+        return float("nan")
+
+
+# ---------------------------------------------------------------------------
+# Plotting helpers
+# ---------------------------------------------------------------------------
+
+
+def _bar_group(
+    ax: plt.Axes,
+    x: np.ndarray,
+    data_per_label: dict[str, list[float]],
+    width: float,
+    colors: list[str],
+    ylabel: str,
+    title: str,
+    concurrency_labels: list[str],
+    fmt: str = ".1f",
+) -> None:
+    n = len(data_per_label)
+    offsets = np.linspace(-(n - 1) * width / 2, (n - 1) * width / 2, n) if n > 1 else [0.0]
+
+    for i, (label, values) in enumerate(data_per_label.items()):
+        plot_vals = [0.0 if math.isnan(v) else v for v in values]
+        bar = ax.bar(x + offsets[i], plot_vals, width, label=label, color=colors[i % len(colors)], alpha=0.85)
+        max_val = max((v for v in values if not math.isnan(v)), default=1.0)
+        for rect, val in zip(bar, values):
+            if not math.isnan(val) and val > 0:
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2,
+                    rect.get_height() + max_val * 0.02,
+                    f"{val:{fmt}}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                    fontweight="bold",
+                )
+
+    ax.set_xlabel("Concurrency", fontsize=11)
+    ax.set_ylabel(ylabel, fontsize=11)
+    ax.set_title(title, fontsize=12, fontweight="bold")
+    ax.set_xticks(x)
+    ax.set_xticklabels(concurrency_labels)
+    ax.legend(fontsize=9)
+    ax.grid(axis="y", alpha=0.3)
+    ax.set_axisbelow(True)
+
+
+COLORS = ["#2196F3", "#FF5722", "#4CAF50", "#FFC107", "#9C27B0"]
+
+
+# ---------------------------------------------------------------------------
+# Comparison plot (multiple labels / runs)
+# ---------------------------------------------------------------------------
+
+
+def plot_comparison(
+    all_runs: list[list[dict]],
+    labels: list[str],
+    output_path: str,
+    task_filter: str | None = None,
+    title_prefix: str = "TTS",
+) -> None:
+    """One 2×2 subplot per task found in the data."""
+    # Determine tasks to plot
+    tasks: list[str] = []
+    for run in all_runs:
+        for r in run:
+            t = r.get("_task", "unknown")
+            if t not in tasks:
+                tasks.append(t)
+    if task_filter:
+        tasks = [t for t in tasks if t == task_filter]
+
+    n_tasks = len(tasks)
+    if n_tasks == 0:
+        print("[plot_results] No tasks found in data.")
+        return
+
+    fig, axes_grid = plt.subplots(n_tasks, 4, figsize=(18, 4.5 * n_tasks))
+    fig.suptitle(f"{title_prefix} Benchmark", fontsize=15, fontweight="bold")
+
+    # Ensure axes_grid is always 2D
+    if n_tasks == 1:
+        axes_grid = [axes_grid]
+
+    for row_idx, task in enumerate(tasks):
+        # Collect concurrencies across all runs for this task
+        all_concs: set[int] = set()
+        for run in all_runs:
+            for r in run:
+                if r.get("_task") == task:
+                    c = r.get("_concurrency")
+                    if c is not None:
+                        all_concs.add(int(c))
+        concurrencies = sorted(all_concs)
+        x = np.arange(len(concurrencies))
+        conc_labels = [str(c) for c in concurrencies]
+
+        def _series(run: list[dict], metric_key: str) -> list[float]:
+            conc_map = {int(r["_concurrency"]): r for r in run if r.get("_task") == task and "_concurrency" in r}
+            return [_get(conc_map.get(c, {}), metric_key) for c in concurrencies]
+
+        metrics = [
+            ("mean_audio_ttfp_ms", "TTFP (ms)", "Time-to-First-Packet", ".0f"),
+            ("mean_e2el_ms", "E2E Latency (ms)", "End-to-End Latency", ".0f"),
+            ("mean_audio_rtf", "RTF", "Real-Time Factor (RTF)", ".3f"),
+            ("audio_throughput", "audio-s / wall-s", "Audio Throughput", ".2f"),
+        ]
+
+        axes_row = axes_grid[row_idx]
+        for col_idx, (key, ylabel, subtitle, fmt) in enumerate(metrics):
+            data_per_label = {lbl: _series(run, key) for lbl, run in zip(labels, all_runs)}
+            _bar_group(
+                axes_row[col_idx],
+                x,
+                data_per_label,
+                width=0.3 if len(labels) > 1 else 0.5,
+                colors=COLORS,
+                ylabel=ylabel,
+                title=f"{task} — {subtitle}",
+                concurrency_labels=conc_labels,
+                fmt=fmt,
+            )
+
+    plt.tight_layout()
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    print(f"Plot saved to {output_path}")
+    plt.close()
+
+
+# ---------------------------------------------------------------------------
+# Markdown comparison table
+# ---------------------------------------------------------------------------
+
+
+def print_comparison_table(all_runs: list[list[dict]], labels: list[str]) -> None:
+    tasks: list[str] = []
+    for run in all_runs:
+        for r in run:
+            t = r.get("_task", "unknown")
+            if t not in tasks:
+                tasks.append(t)
+
+    perf_metrics = [
+        ("TTFP (ms)", "mean_audio_ttfp_ms", ".1f"),
+        ("E2E (ms)", "mean_e2el_ms", ".1f"),
+        ("RTF", "mean_audio_rtf", ".3f"),
+        ("Throughput (a-s/s)", "audio_throughput", ".2f"),
+    ]
+    quality_metrics = [
+        ("WER (%)", "seed_tts_mean_wer", ".1f"),
+        ("SIM", "seed_tts_mean_sim", ".3f"),
+        ("UTMOS", "seed_tts_mean_utmos", ".2f"),
+    ]
+
+    for task in tasks:
+        all_concs: set[int] = set()
+        for run in all_runs:
+            for r in run:
+                if r.get("_task") == task:
+                    c = r.get("_concurrency")
+                    if c is not None:
+                        all_concs.add(int(c))
+        concurrencies = sorted(all_concs)
+
+        print(f"\n## {task}\n")
+        col_header = "| Metric | Concurrency |" + "".join(f" {lbl} |" for lbl in labels)
+        sep = "| --- | --- |" + " --- |" * len(labels)
+        print(col_header)
+        print(sep)
+
+        for metric, key, fmt in perf_metrics + quality_metrics:
+            for c in concurrencies:
+                row = f"| {metric} | {c} |"
+                for run in all_runs:
+                    conc_map = {
+                        int(r["_concurrency"]): r for r in run if r.get("_task") == task and "_concurrency" in r
+                    }
+                    val = _get(conc_map.get(c, {}), key)
+                    row += f" {val:{fmt}} |" if not math.isnan(val) else " n/a |"
+                print(row)
+
+        # Improvement column (2-run comparison only)
+        if len(all_runs) == 2:
+            print(f"\n### Improvement ({labels[0]} vs {labels[1]})\n")
+            print("| Metric | Concurrency | Change |")
+            print("| --- | --- | --- |")
+            for metric, key, _ in perf_metrics:
+                for c in concurrencies:
+                    conc_map0 = {
+                        int(r["_concurrency"]): r for r in all_runs[0] if r.get("_task") == task and "_concurrency" in r
+                    }
+                    conc_map1 = {
+                        int(r["_concurrency"]): r for r in all_runs[1] if r.get("_task") == task and "_concurrency" in r
+                    }
+                    v0 = _get(conc_map0.get(c, {}), key)
+                    v1 = _get(conc_map1.get(c, {}), key)
+                    if not math.isnan(v0) and not math.isnan(v1) and v1 > 0:
+                        pct = (v1 - v0) / v1 * 100
+                        print(f"| {metric} | {c} | {pct:+.1f}% |")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--results",
+        type=str,
+        nargs="+",
+        action="append",
+        required=True,
+        metavar="FILE",
+        help="JSON result file(s) for one run. Repeat --results for multiple runs to compare.",
+    )
+    parser.add_argument(
+        "--labels",
+        type=str,
+        nargs="+",
+        default=None,
+        help="Label for each --results group (must match the number of --results groups).",
+    )
+    parser.add_argument("--output", type=str, default="results/tts_benchmark.png", help="Output image path.")
+    parser.add_argument("--title", type=str, default="TTS", help="Title prefix for the plot.")
+    parser.add_argument("--task", type=str, default=None, help="Filter to a single task (e.g. voice_clone).")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    # args.results is a list-of-lists due to action="append"
+    all_runs: list[list[dict]] = [load_run(group) for group in args.results]
+    n_runs = len(all_runs)
+
+    labels: list[str]
+    if args.labels:
+        if len(args.labels) != n_runs:
+            raise SystemExit(f"--labels count ({len(args.labels)}) must match --results groups ({n_runs})")
+        labels = args.labels
+    else:
+        labels = [f"run{i + 1}" for i in range(n_runs)]
+
+    print_comparison_table(all_runs, labels)
+    plot_comparison(all_runs, labels, args.output, task_filter=args.task, title_prefix=args.title)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/voxcpm/README.md

@@ -1,119 +0,0 @@
-# VoxCPM Benchmark
-
-This directory contains both:
-
-- online serving benchmark through the OpenAI-compatible `/v1/audio/speech` API
-- offline benchmark for `Omni` / `AsyncOmni`
-- full offline smoke-matrix orchestration
-
-Both benchmark paths report:
-
-- TTFP: time to first PCM packet
-- E2E latency
-- RTF: real-time factor (`e2e / audio_duration`)
-
-## Offline Benchmark
-
-Single offline benchmark run:
-
-```bash
-python benchmarks/voxcpm/vllm_omni/bench_tts_offline.py \
-    --model /path/to/voxcpm-model \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm.yaml \
-    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni." \
-    --warmup-runs 1 \
-    --output-dir benchmarks/voxcpm/results/offline_single
-```
-
-Streaming offline benchmark:
-
-```bash
-python benchmarks/voxcpm/vllm_omni/bench_tts_offline.py \
-    --model /path/to/voxcpm-model \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
-    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni." \
-    --warmup-runs 1 \
-    --output-dir benchmarks/voxcpm/results/offline_streaming
-```
-
-Full fixed offline matrix, equivalent to the old `examples/offline_inference/voxcpm/test.py`:
-
-```bash
-python benchmarks/voxcpm/vllm_omni/run_offline_matrix.py \
-    --model /path/to/voxcpm-model \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "The exact transcript spoken in reference.wav." \
-    --output-root benchmarks/voxcpm/results/offline_matrix
-```
-
-The full matrix covers both routes:
-
-- streaming: `voxcpm_async_chunk.yaml`
-- sync: `voxcpm.yaml`
-
-And these six scenarios under each route:
-
-- warmup + single TTS
-- warmup + single voice cloning
-- warmup + batch TTS
-- warmup + batch voice cloning
-- cold single TTS
-- cold single voice cloning
-
-`bench_tts_offline.py` itself no longer writes `summary.json` / `results.json`; it prints TTFP / RTF inline and saves generated WAV files only. The matrix runner keeps only per-case `run.log`.
-
-## Start the Server
-
-Async-chunk:
-
-```bash
-vllm serve /path/to/voxcpm-model \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni \
-    --port 8091
-```
-
-Non-streaming:
-
-```bash
-vllm serve /path/to/voxcpm-model \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm.yaml \
-    --trust-remote-code \
-    --enforce-eager \
-    --omni \
-    --port 8091
-```
-
-## Run the Benchmark
-
-```bash
-python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
-    --host 127.0.0.1 \
-    --port 8091 \
-    --num-prompts 20 \
-    --max-concurrency 1 \
-    --result-dir /tmp/voxcpm_bench
-```
-
-Voice cloning benchmark:
-
-```bash
-python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
-    --host 127.0.0.1 \
-    --port 8091 \
-    --num-prompts 10 \
-    --max-concurrency 1 \
-    --ref-audio https://example.com/reference.wav \
-    --ref-text "The exact transcript spoken in the reference audio." \
-    --result-dir /tmp/voxcpm_clone_bench
-```
-
-## Notes
-
-- The benchmark uses `stream=true` and `response_format=pcm` so TTFP is measured from the first audio packet.
-- `RTF < 1.0` means the server generates audio faster than real time.
-- For `voxcpm_async_chunk.yaml`, keep concurrency at `1`. This matches native VoxCPM streaming more closely.
-- Do not benchmark concurrent online streaming on `voxcpm_async_chunk.yaml`; use `voxcpm.yaml` for multi-request throughput runs.
-- For the offline matrix mode, `--ref-audio` and `--ref-text` are required because clone cases are part of the fixed coverage set.

benchmarks/voxcpm/vllm_omni/bench_tts_serve.py

@@ -1,283 +0,0 @@
-"""Benchmark VoxCPM via /v1/audio/speech.
-
-Reports TTFP (time to first packet), E2E latency, and RTF (real-time factor).
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import time
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from pathlib import Path
-
-import aiohttp
-import numpy as np
-from tqdm.asyncio import tqdm
-
-DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
-DEFAULT_SAMPLE_RATE = 24000
-PROMPTS = [
-    "Hello, welcome to the VoxCPM speech benchmark.",
-    "This is a short benchmark prompt for online text-to-speech generation.",
-    "The quick brown fox jumps over the lazy dog near the riverbank.",
-    "Please remember to bring your identification documents tomorrow morning.",
-    "Learning a new language takes patience, practice, and curiosity.",
-    "This benchmark reports TTFP and RTF for the VoxCPM online serving path.",
-]
-
-
-@dataclass
-class RequestResult:
-    success: bool = False
-    ttfp: float = 0.0
-    e2e: float = 0.0
-    audio_bytes: int = 0
-    audio_duration: float = 0.0
-    rtf: float = 0.0
-    prompt: str = ""
-    error: str = ""
-
-
-@dataclass
-class BenchmarkResult:
-    concurrency: int = 0
-    num_prompts: int = 0
-    completed: int = 0
-    failed: int = 0
-    duration_s: float = 0.0
-    mean_ttfp_ms: float = 0.0
-    median_ttfp_ms: float = 0.0
-    p95_ttfp_ms: float = 0.0
-    mean_e2e_ms: float = 0.0
-    median_e2e_ms: float = 0.0
-    p95_e2e_ms: float = 0.0
-    mean_rtf: float = 0.0
-    median_rtf: float = 0.0
-    p95_rtf: float = 0.0
-    total_audio_duration_s: float = 0.0
-    request_throughput: float = 0.0
-    per_request: list[dict[str, float | str]] = field(default_factory=list)
-
-
-def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = DEFAULT_SAMPLE_RATE, sample_width: int = 2) -> float:
-    num_samples = num_bytes / sample_width
-    return num_samples / sample_rate
-
-
-async def send_tts_request(
-    session: aiohttp.ClientSession,
-    api_url: str,
-    *,
-    model: str,
-    prompt: str,
-    ref_audio: str | None,
-    ref_text: str | None,
-    pbar: tqdm | None = None,
-) -> RequestResult:
-    payload: dict[str, object] = {
-        "model": model,
-        "input": prompt,
-        "stream": True,
-        "response_format": "pcm",
-    }
-    if ref_audio is not None:
-        payload["ref_audio"] = ref_audio
-    if ref_text is not None:
-        payload["ref_text"] = ref_text
-
-    result = RequestResult(prompt=prompt)
-    started_at = time.perf_counter()
-
-    try:
-        async with session.post(api_url, json=payload) as response:
-            if response.status != 200:
-                result.error = f"HTTP {response.status}: {await response.text()}"
-                return result
-
-            first_chunk = True
-            total_bytes = 0
-            async for chunk in response.content.iter_any():
-                if not chunk:
-                    continue
-                if first_chunk:
-                    result.ttfp = time.perf_counter() - started_at
-                    first_chunk = False
-                total_bytes += len(chunk)
-
-            result.e2e = time.perf_counter() - started_at
-            result.audio_bytes = total_bytes
-            result.audio_duration = pcm_bytes_to_duration(total_bytes)
-            if result.audio_duration > 0:
-                result.rtf = result.e2e / result.audio_duration
-            result.success = True
-    except Exception as e:
-        result.error = str(e)
-        result.e2e = time.perf_counter() - started_at
-
-    if pbar is not None:
-        pbar.update(1)
-    return result
-
-
-async def run_benchmark(
-    *,
-    host: str,
-    port: int,
-    model: str,
-    num_prompts: int,
-    max_concurrency: int,
-    num_warmups: int,
-    ref_audio: str | None,
-    ref_text: str | None,
-) -> BenchmarkResult:
-    api_url = f"http://{host}:{port}/v1/audio/speech"
-    connector = aiohttp.TCPConnector(limit=max_concurrency, limit_per_host=max_concurrency, keepalive_timeout=60)
-    timeout = aiohttp.ClientTimeout(total=600)
-
-    async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
-        if num_warmups > 0:
-            print(f"  Warming up with {num_warmups} requests...")
-            warmup_tasks = [
-                send_tts_request(
-                    session,
-                    api_url,
-                    model=model,
-                    prompt=PROMPTS[i % len(PROMPTS)],
-                    ref_audio=ref_audio,
-                    ref_text=ref_text,
-                )
-                for i in range(num_warmups)
-            ]
-            await asyncio.gather(*warmup_tasks)
-            print("  Warmup done.")
-
-        request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
-        semaphore = asyncio.Semaphore(max_concurrency)
-        pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
-
-        async def limited_request(prompt: str) -> RequestResult:
-            async with semaphore:
-                return await send_tts_request(
-                    session,
-                    api_url,
-                    model=model,
-                    prompt=prompt,
-                    ref_audio=ref_audio,
-                    ref_text=ref_text,
-                    pbar=pbar,
-                )
-
-        started_at = time.perf_counter()
-        results = await asyncio.gather(*[asyncio.create_task(limited_request(prompt)) for prompt in request_prompts])
-        duration = time.perf_counter() - started_at
-        pbar.close()
-
-    succeeded = [result for result in results if result.success]
-    bench = BenchmarkResult(
-        concurrency=max_concurrency,
-        num_prompts=num_prompts,
-        completed=len(succeeded),
-        failed=len(results) - len(succeeded),
-        duration_s=duration,
-    )
-
-    if not succeeded:
-        return bench
-
-    ttfps = np.array([result.ttfp * 1000 for result in succeeded], dtype=np.float64)
-    e2es = np.array([result.e2e * 1000 for result in succeeded], dtype=np.float64)
-    rtfs = np.array([result.rtf for result in succeeded], dtype=np.float64)
-    audio_durations = np.array([result.audio_duration for result in succeeded], dtype=np.float64)
-
-    bench.mean_ttfp_ms = float(np.mean(ttfps))
-    bench.median_ttfp_ms = float(np.median(ttfps))
-    bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
-    bench.mean_e2e_ms = float(np.mean(e2es))
-    bench.median_e2e_ms = float(np.median(e2es))
-    bench.p95_e2e_ms = float(np.percentile(e2es, 95))
-    bench.mean_rtf = float(np.mean(rtfs))
-    bench.median_rtf = float(np.median(rtfs))
-    bench.p95_rtf = float(np.percentile(rtfs, 95))
-    bench.total_audio_duration_s = float(np.sum(audio_durations))
-    bench.request_throughput = len(succeeded) / duration if duration > 0 else 0.0
-    bench.per_request = [
-        {
-            "prompt": result.prompt,
-            "ttfp_ms": result.ttfp * 1000,
-            "e2e_ms": result.e2e * 1000,
-            "rtf": result.rtf,
-            "audio_duration_s": result.audio_duration,
-        }
-        for result in succeeded
-    ]
-
-    return bench
-
-
-def print_summary(result: BenchmarkResult) -> None:
-    width = 54
-    print("")
-    print("=" * width)
-    print(f"{'VoxCPM Serving Benchmark':^{width}}")
-    print("=" * width)
-    print(f"concurrency         : {result.concurrency}")
-    print(f"requests            : {result.completed}/{result.num_prompts} succeeded")
-    print(f"wall time (s)       : {result.duration_s:.3f}")
-    print(f"mean TTFP (ms)      : {result.mean_ttfp_ms:.2f}")
-    print(f"p95 TTFP (ms)       : {result.p95_ttfp_ms:.2f}")
-    print(f"mean E2E (ms)       : {result.mean_e2e_ms:.2f}")
-    print(f"p95 E2E (ms)        : {result.p95_e2e_ms:.2f}")
-    print(f"mean RTF            : {result.mean_rtf:.3f}")
-    print(f"p95 RTF             : {result.p95_rtf:.3f}")
-    print(f"request throughput  : {result.request_throughput:.2f} req/s")
-    print("=" * width)
-
-
-async def main_async(args) -> None:
-    result_dir = Path(args.result_dir)
-    result_dir.mkdir(parents=True, exist_ok=True)
-
-    all_results: list[BenchmarkResult] = []
-    for concurrency in args.max_concurrency:
-        result = await run_benchmark(
-            host=args.host,
-            port=args.port,
-            model=args.model,
-            num_prompts=args.num_prompts,
-            max_concurrency=concurrency,
-            num_warmups=args.num_warmups,
-            ref_audio=args.ref_audio,
-            ref_text=args.ref_text,
-        )
-        print_summary(result)
-        all_results.append(result)
-
-    payload = {
-        "model": args.model,
-        "created_at": datetime.utcnow().isoformat() + "Z",
-        "results": [asdict(result) for result in all_results],
-    }
-    result_path = result_dir / "bench_tts_serve.json"
-    result_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
-    print(f"Saved results to: {result_path}")
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Benchmark VoxCPM via /v1/audio/speech")
-    parser.add_argument("--host", default="127.0.0.1", help="Server host")
-    parser.add_argument("--port", type=int, default=8091, help="Server port")
-    parser.add_argument("--model", default=DEFAULT_MODEL, help="Model name or path")
-    parser.add_argument("--num-prompts", type=int, default=20, help="Number of prompts to send")
-    parser.add_argument("--max-concurrency", type=int, nargs="+", default=[1], help="Concurrency levels to benchmark")
-    parser.add_argument("--num-warmups", type=int, default=3, help="Warmup request count")
-    parser.add_argument("--ref-audio", default=None, help="Reference audio URL or data URL for voice cloning")
-    parser.add_argument("--ref-text", default=None, help="Reference audio transcript for voice cloning")
-    parser.add_argument("--result-dir", default="results", help="Directory to save benchmark JSON")
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    asyncio.run(main_async(parse_args()))

benchmarks/voxcpm/vllm_omni/run_offline_matrix.py

@@ -1,303 +0,0 @@
-"""Run the full offline VoxCPM smoke matrix.
-
-This script keeps the old `test.py` coverage, but delegates each case to
-`bench_tts_offline.py` so the benchmark runner itself stays focused on a
-single execution path.
-"""
-
-from __future__ import annotations
-
-import shlex
-import subprocess
-import sys
-import time
-from dataclasses import dataclass
-from pathlib import Path
-
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-REPO_ROOT = Path(__file__).resolve().parents[3]
-BENCH_SCRIPT = Path(__file__).with_name("bench_tts_offline.py")
-DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
-DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
-DEFAULT_OUTPUT_ROOT = BENCH_SCRIPT.parents[1] / "results" / "offline_matrix"
-
-SINGLE_TTS_TEXT = "This is a single text-to-speech smoke test for VoxCPM on vLLM Omni."
-SINGLE_CLONE_TEXT = "This sentence is synthesized with the cloned voice for validation."
-BATCH_TTS_TEXTS = [
-    "The first batch text-to-speech sample validates sequential batch execution.",
-    "The second batch text-to-speech sample checks another prompt in the same file.",
-    "The third batch text-to-speech sample completes the sequential batch path.",
-]
-BATCH_CLONE_TEXTS = [
-    "The first cloned sample validates sequential batch voice cloning.",
-    "The second cloned sample checks the same reference voice on another prompt.",
-    "The third cloned sample finishes the shared-reference clone batch path.",
-]
-
-
-@dataclass(frozen=True, slots=True)
-class ModeSpec:
-    name: str
-    stage_config: Path
-
-
-@dataclass(frozen=True, slots=True)
-class CaseSpec:
-    name: str
-    warmup_runs: int
-    prompt_kind: str
-    voice_clone: bool
-
-
-@dataclass(frozen=True, slots=True)
-class CaseResult:
-    mode: str
-    case: str
-    returncode: int
-    elapsed_s: float
-    output_dir: Path
-    log_path: Path
-
-    @property
-    def ok(self) -> bool:
-        return self.returncode == 0
-
-
-MODE_SPECS = [
-    ModeSpec(name="streaming", stage_config=DEFAULT_STAGE_ASYNC),
-    ModeSpec(name="sync", stage_config=DEFAULT_STAGE_SYNC),
-]
-
-CASE_SPECS = [
-    CaseSpec(name="warmup_single_tts", warmup_runs=1, prompt_kind="single", voice_clone=False),
-    CaseSpec(name="warmup_single_clone", warmup_runs=1, prompt_kind="single", voice_clone=True),
-    CaseSpec(name="warmup_batch_tts", warmup_runs=1, prompt_kind="batch", voice_clone=False),
-    CaseSpec(name="warmup_batch_clone", warmup_runs=1, prompt_kind="batch", voice_clone=True),
-    CaseSpec(name="cold_single_tts", warmup_runs=0, prompt_kind="single", voice_clone=False),
-    CaseSpec(name="cold_single_clone", warmup_runs=0, prompt_kind="single", voice_clone=True),
-]
-
-
-def _write_lines(path: Path, lines: list[str]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-
-def _prepare_batch_inputs(output_root: Path) -> tuple[Path, Path]:
-    input_dir = output_root / "inputs"
-    batch_tts_path = input_dir / "batch_tts_prompts.txt"
-    batch_clone_path = input_dir / "batch_clone_prompts.txt"
-    _write_lines(batch_tts_path, BATCH_TTS_TEXTS)
-    _write_lines(batch_clone_path, BATCH_CLONE_TEXTS)
-    return batch_tts_path, batch_clone_path
-
-
-def _base_command(args, mode: ModeSpec, output_dir: Path) -> list[str]:
-    cmd = [
-        args.python,
-        str(BENCH_SCRIPT),
-        "--model",
-        args.model,
-        "--stage-configs-path",
-        str(mode.stage_config),
-        "--output-dir",
-        str(output_dir),
-        "--num-runs",
-        str(args.num_runs),
-        "--stage-init-timeout",
-        str(args.stage_init_timeout),
-    ]
-    cmd.append("--log-stats" if args.log_stats else "--no-log-stats")
-    cmd.extend(["--cfg-value", str(args.cfg_value)])
-    cmd.extend(["--inference-timesteps", str(args.inference_timesteps)])
-    cmd.extend(["--min-len", str(args.min_len)])
-    cmd.extend(["--max-new-tokens", str(args.max_new_tokens)])
-    if args.streaming_prefix_len is not None:
-        cmd.extend(["--streaming-prefix-len", str(args.streaming_prefix_len)])
-    if args.enable_profiler:
-        profiler_dir = Path(args.profiler_dir) if args.profiler_dir is not None else (output_dir / "profiler")
-        cmd.append("--enable-profiler")
-        cmd.extend(["--profiler-dir", str(profiler_dir)])
-        cmd.extend(["--profiler-wait-seconds", str(args.profiler_wait_seconds)])
-        if args.profiler_stages is not None:
-            cmd.append("--profiler-stages")
-            cmd.extend(str(stage_id) for stage_id in args.profiler_stages)
-    return cmd
-
-
-def _build_case_command(
-    args,
-    mode: ModeSpec,
-    case: CaseSpec,
-    *,
-    batch_tts_path: Path,
-    batch_clone_path: Path,
-    output_dir: Path,
-) -> list[str]:
-    cmd = _base_command(args, mode, output_dir)
-    cmd.extend(["--warmup-runs", str(case.warmup_runs)])
-    if case.prompt_kind == "single":
-        cmd.extend(["--text", SINGLE_CLONE_TEXT if case.voice_clone else SINGLE_TTS_TEXT])
-    else:
-        cmd.extend(["--txt-prompts", str(batch_clone_path if case.voice_clone else batch_tts_path)])
-    if case.voice_clone:
-        cmd.extend(["--ref-audio", args.ref_audio, "--ref-text", args.ref_text])
-    return cmd
-
-
-def _run_case(
-    args,
-    mode: ModeSpec,
-    case: CaseSpec,
-    *,
-    batch_tts_path: Path,
-    batch_clone_path: Path,
-    output_root: Path,
-) -> CaseResult:
-    case_output_dir = output_root / mode.name / case.name
-    case_output_dir.mkdir(parents=True, exist_ok=True)
-    case_log_path = case_output_dir / "run.log"
-    cmd = _build_case_command(
-        args,
-        mode,
-        case,
-        batch_tts_path=batch_tts_path,
-        batch_clone_path=batch_clone_path,
-        output_dir=case_output_dir,
-    )
-
-    print()
-    print("=" * 80)
-    print(f"[{mode.name}] {case.name}")
-    print(f"Output directory: {case_output_dir}")
-    print(shlex.join(cmd))
-
-    start = time.perf_counter()
-    with case_log_path.open("w", encoding="utf-8") as log_fp:
-        process = subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-            bufsize=1,
-        )
-        assert process.stdout is not None
-        for line in process.stdout:
-            print(line, end="")
-            log_fp.write(line)
-        process.wait()
-
-    elapsed_s = time.perf_counter() - start
-    status = "PASS" if (process.returncode or 0) == 0 else f"FAIL({process.returncode})"
-    print(f"[{mode.name}] {case.name} -> {status} ({elapsed_s:.2f}s)")
-    return CaseResult(
-        mode=mode.name,
-        case=case.name,
-        returncode=int(process.returncode or 0),
-        elapsed_s=elapsed_s,
-        output_dir=case_output_dir,
-        log_path=case_log_path,
-    )
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Run the full offline VoxCPM smoke matrix.")
-    parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
-    parser.add_argument("--ref-audio", type=str, required=True, help="Reference audio path for clone cases.")
-    parser.add_argument("--ref-text", type=str, required=True, help="Exact transcript spoken in --ref-audio.")
-    parser.add_argument("--output-root", type=str, default=str(DEFAULT_OUTPUT_ROOT), help="Root directory for outputs.")
-    parser.add_argument("--python", type=str, default=sys.executable, help="Python executable used to launch cases.")
-    parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
-    parser.add_argument("--log-stats", dest="log_stats", action="store_true", help="Enable vLLM Omni stats logging.")
-    parser.add_argument(
-        "--no-log-stats",
-        dest="log_stats",
-        action="store_false",
-        help="Disable vLLM Omni stats logging.",
-    )
-    parser.set_defaults(log_stats=True)
-    parser.add_argument("--num-runs", type=int, default=1, help="Number of measured runs per case.")
-    parser.add_argument("--cfg-value", type=float, default=2.0, help="Classifier-free guidance value for VoxCPM.")
-    parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of inference timesteps.")
-    parser.add_argument("--min-len", type=int, default=2, help="Minimum generated token length.")
-    parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum generated token length.")
-    parser.add_argument(
-        "--streaming-prefix-len",
-        type=int,
-        default=None,
-        help="Optional VoxCPM streaming window passed to streaming cases.",
-    )
-    parser.add_argument("--enable-profiler", action="store_true", help="Enable torch profiler for each case.")
-    parser.add_argument(
-        "--profiler-dir",
-        type=str,
-        default=None,
-        help="Profiler output root. Defaults to <case-output-dir>/profiler.",
-    )
-    parser.add_argument(
-        "--profiler-stages",
-        type=int,
-        nargs="*",
-        default=None,
-        help="Optional stage ids to profile. Defaults to all configured stages.",
-    )
-    parser.add_argument(
-        "--profiler-wait-seconds",
-        type=float,
-        default=30.0,
-        help="Seconds to wait after stopping profiler for traces to flush.",
-    )
-    args = parser.parse_args()
-    if args.num_runs < 1:
-        parser.error("--num-runs must be >= 1")
-    return args
-
-
-def main(args) -> int:
-    output_root = Path(args.output_root)
-    output_root.mkdir(parents=True, exist_ok=True)
-    batch_tts_path, batch_clone_path = _prepare_batch_inputs(output_root)
-
-    print(f"Model: {args.model}")
-    print(f"Reference audio: {args.ref_audio}")
-    print(f"Reference text: {args.ref_text}")
-    print(f"Python: {args.python}")
-    print(f"Output root: {output_root}")
-    print(f"Cases: {len(MODE_SPECS) * len(CASE_SPECS)}")
-
-    results: list[CaseResult] = []
-    for mode in MODE_SPECS:
-        for case in CASE_SPECS:
-            results.append(
-                _run_case(
-                    args,
-                    mode,
-                    case,
-                    batch_tts_path=batch_tts_path,
-                    batch_clone_path=batch_clone_path,
-                    output_root=output_root,
-                )
-            )
-
-    failed = [result for result in results if not result.ok]
-    print()
-    print("=" * 80)
-    print("Summary:")
-    for result in results:
-        status = "PASS" if result.ok else f"FAIL({result.returncode})"
-        print(f"- [{result.mode}] {result.case}: {status} ({result.elapsed_s:.2f}s)")
-        print(f"  output_dir={result.output_dir}")
-        print(f"  log={result.log_path}")
-
-    print(f"Passed: {len(results) - len(failed)}/{len(results)}")
-    if failed:
-        print("Failed cases:")
-        for result in failed:
-            print(f"- [{result.mode}] {result.case}: see {result.log_path}")
-        return 1
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main(parse_args()))

docs/.nav.yml

@@ -22,6 +22,7 @@ nav:
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
       - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
       - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
+      - InternVLA-A1: user_guide/examples/offline_inference/internvla_a1.md
       - MammothModa2-Preview: user_guide/examples/offline_inference/mammothmodal2_preview.md
       - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md

docs/cli/serve.md

@@ -12,9 +12,11 @@ The stage-based CLI is designed for deployments that require launching each pipe
   `vllm_omni/model_executor/stage_configs/`, the `--stage-configs-path` parameter remains mandatory.
 
 Example: Initializing Stage 0 (Orchestrator and API Server):
+The commands below show a common device mapping where Stage 0 uses GPU 0 and
+worker stages use GPU 1 via `CUDA_VISIBLE_DEVICES`.
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
@@ -24,7 +26,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 Example: Initializing a Headless Worker Stage (Stage 1):
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -46,7 +48,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
 the stage-based CLI permits the direct initialization of Stage 1 with explicit parameters:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \

docs/configuration/stage_configs.md

@@ -19,7 +19,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | `platforms` | dict | optional | `null` | Keyed by `npu` / `rocm` / `xpu`, each contains a `stages:` list with per-platform overrides applied on top of the CUDA defaults. |
 | `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only`). |
 | `trust_remote_code` | bool | optional | `true` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage. |
-| `distributed_executor_backend` | str | optional | `"mp"` | **Pipeline-wide.** Executor backend (`"mp"` or `"ray"`). |
+| `distributed_executor_backend` | str \| null | optional | `null` | **Pipeline-wide.** Distributed executor backend forwarded to vLLM (`"mp"`, `"ray"`, `"external_launcher"`). If omitted, vLLM auto-selects backend from runtime topology. |
 | `dtype` | str \| null | optional | `null` | **Pipeline-wide.** Model dtype for every stage. |
 | `quantization` | str \| null | optional | `null` | **Pipeline-wide.** Quantization method for every stage. |
 | `enable_prefix_caching` | bool | optional | `false` | **Pipeline-wide.** Prefix cache toggle applied to every stage. |
@@ -27,6 +27,9 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | `data_parallel_size` | int | optional | `1` | **Pipeline-wide.** DP degree for every stage. |
 | `pipeline_parallel_size` | int | optional | `1` | **Pipeline-wide.** PP degree for every stage. |
 
+Note: for diffusion path, `distributed_executor_backend` currently defaults to
+`mp`, and `ray` / `external_launcher` are not fully supported yet.
+
 ### Stage fields
 
 Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine.
@@ -99,15 +102,17 @@ The stage-based CLI paradigm facilitates the execution of discrete pipeline stag
 
 For migrated architectures, the system automatically resolves and loads the bundled deployment YAML. Consequently, the primary execution path
 does **not** necessitate the explicit definition of `--deploy-config`:
+the example below uses `CUDA_VISIBLE_DEVICES=0` for Stage 0 and
+`CUDA_VISIBLE_DEVICES=1` for Stage 1.
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
     --omni-master-port 26000
 
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -129,7 +134,7 @@ Conversely, in the context of the **stage-based CLI** paradigm, given that each
 can be defined uniformly via explicit CLI flags on the corresponding instantiation command, rendering composite `--stage-overrides` JSON strings unnecessary:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \
@@ -186,7 +191,7 @@ Within the stage-based CLI paradigm, equivalent configuration parameters can inh
 as command-line arguments to the designated single-stage process instantiation:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 0 \
     --max-num-seqs 8 \
     --omni-master-address 127.0.0.1 \

docs/contributing/ci/CI_5levels.md

@@ -108,7 +108,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
         /tests/dfx/stability/tests/test_qwen3_omni.json<br>
         /tests/dfx/stability/tests/test_wan22.json<br>
         <strong>Reliability:</strong><br>
-        tests/e2e/reliability/test_{model_name}.py
+        tests/dfx/reliability/test_reliability_{model_key}.py<br>
+        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>)
       </td>
       <td>
         <a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
@@ -576,12 +577,17 @@ L5 level testing focuses on the performance of model services under ***long-runn
 ### 4.2 Testing Content and Scope
 
 -   ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
--   ***Reliability Testing***: Uses `tests/e2e/reliability/test_{model_name}.py` to actively simulate various fault and abnormal scenarios, such as: dependent service interruption, abnormal input data, network flicker, hardware resource preemption, etc., to verify the system's fault tolerance, self-healing, and graceful degradation capabilities.
+-   ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
 
 ### 4.3 Test Directory and Execution Files
 
 -   ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
--   ***Reliability Test Suite***: `tests/e2e/reliability/test_{model_name}.py`
+-   ***Reliability Test Suite*** (`tests/dfx/reliability/`):
+    -   `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default).
+    -   `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
+    -   `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`).
+    -   `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
+    -   `README.md` — Short local run commands for this directory.
 
 ### 4.4 Execution Method and Example
 
@@ -662,8 +668,57 @@ All other optional parameters follow the same rules as the in Chapter 3.4.
 
 </details>
 
+<details>
+<summary> Reliability test suite (<code>tests/dfx/reliability</code>)</summary>
+
+#### Purpose and relationship to stability
+
+Reliability tests are **short fault-injection** integration runs (L5 **(b)** in `tests/dfx/reliability/README.md`). They complement **stability** JSON-driven long runs: instead of hours of steady traffic, they **perturb** the server (GPU OOM sidecar, fatal signals on selected processes) and check **failure mode** and **latency bounds** (e.g. chat or `/v1/videos` must not hang under concurrent fault-time load).
+
+#### Directory layout
+
+| Path | Responsibility |
+| ---- | -------------- |
+| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). |
+| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). |
+| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). |
+| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
+| `README.md` | Minimal run / collect examples. |
+
+#### Parametrization and markers
+
+- Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
+- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card).
+- **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
+
+#### CI trigger
+
+Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example:
+
+```bash
+pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
+pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
+```
+
+#### Local commands
+
+```bash
+pytest --collect-only tests/dfx/reliability
+pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
+```
+
+#### Adding a new model suite
+
+1. Add `test_reliability_<model>.py` under `tests/dfx/reliability/`.
+2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files).
+3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML.
+4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/test-weekly.yml`** when the suite should run in weekly L5.
+
+</details>
+
 -   -   ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
-    -   ***Reliability***: `pytest -s -v tests/e2e/reliability/test_{model_name}.py`
+    -   ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_<suite>.py` for new models)
 
 ## Summary
 

docs/models/supported_models.md

@@ -20,6 +20,7 @@ th {
 | `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
+| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -195,6 +195,7 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
 | StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
 | Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
 | Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |
 
 **Notes:**
 - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally

docs/user_guide/diffusion/quantization/msmodelslim.md

@@ -0,0 +1,56 @@
+# msModelSlim Quantization
+
+## Overview
+
+[msModelSlim](https://github.com/Ascend/msmodelslim) is an Ascend-friendly compression tool focused on acceleration, using compression techniques, and built for Ascend hardware. It includes a series of inference optimization technologies such as quantization and compression, aiming to accelerate large language dense models, MoE models, multimodal understanding models, multimodal generation models, etc.
+
+Once you have a quantized model which is generated by **msModelSlim**, you can use vLLM Omni for inference by specifying the --quantization ascend parameter to enable quantization features.
+
+### Supported Schemes
+
+| Scheme | Bits | Status |
+|--------|------|--------|
+| W8A8 | 8 | ✅ Supported |
+| W4A4 | 4 | Planned |
+
+W8A8 is the first supported scheme. Additional schemes will be added in future releases.
+
+## Model Quantization
+
+The following example shows how to generate W8A8 quantized weights for the [Wan2_2 model](https://gitcode.com/Ascend/msmodelslim/blob/master/example/multimodal_sd/Wan2_2/README.md).
+
+**Quantization Script:**
+
+```bash
+msmodelslim quant \
+  --model_path /path/to/wan2_2_t2v_float_weights \
+  --save_path /path/to/wan2_2_t2v_quantized_weights \
+  --device npu \
+  --model_type Wan2_2 \
+  --config_path /lab_practice/wan2_2/wan2_2_w8a8f8_mxfp_t2v.yaml \
+  --trust_remote_code True
+```
+
+After quantization completes, the output directory will contain the quantized model files.
+
+For more examples, refer to the [official examples](https://gitcode.com/Ascend/msit/tree/master/msmodelslim/example).
+
+## Configuration
+
+1. **CLI**: pass `--quantization ascend`.
+
+```bash
+# Offline inference
+python text_to_image.py --model <your-model> --quantization ascend
+
+# Online serving
+vllm serve <your-model> --omni --quantization ascend
+```
+
+## Supported Models
+
+| Model | HF Models | Recommendation | `ignored_layers` |
+|-------|-----------|---------------|------------------|
+| HunyuanImage-3.0 | - | All layers | None |
+
+Currently, quantized HunyuanImage-3.0 weights have not been uploaded to public model platforms such as Hugging Face. You can use a [HunyuanImage-3.0-adapted msModelSlim version](https://gitcode.com/betta18/msmodelslim/tree/hyimage3_mxfp8) to generate the quantized weights manually. We will upload the quantized weights as soon as possible.

docs/user_guide/diffusion_features.md

@@ -107,8 +107,8 @@ The following tables show which models support each feature:
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **FLUX.1-dev** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
+| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅  | ❌ | ❌ | ❌ |
+| **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |

docs/user_guide/examples/offline_inference/glm_image.md

@@ -1,154 +1,87 @@
-# GLM-Image Multistage End-to-End Inference
+# GLM-Image Offline Inference
 
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/glm_image>.
-
-
-This example demonstrates how to run GLM-Image with the vLLM-Omni multistage architecture.
+GLM-Image is a 2-stage image generation model (AR + Diffusion) supported by vLLM-Omni's
+declarative config system. The pipeline topology and stage structure are declared in
+`vllm_omni/model_executor/models/glm_image/pipeline.py`; deployment knobs live in
+`vllm_omni/deploy/glm_image.yaml`.
 
 ## Architecture
 
-GLM-Image uses a 2-stage pipeline:
-
 ```
-┌─────────────────────────────────────────────────────────────┐
-│                     GLM-Image Pipeline                       │
-├─────────────────────────────────────────────────────────────┤
-│                                                             │
-│  Stage 0 (AR Model)              Stage 1 (Diffusion)        │
-│  ┌─────────────────┐            ┌─────────────────────┐     │
-│  │ vLLM-optimized  │            │  GlmImagePipeline   │     │
-│  │ GlmImageFor     │  prior     │  ┌───────────────┐  │     │
-│  │ Conditional     │──tokens───►│  │ DiT Denoiser  │  │     │
-│  │ Generation      │            │  └───────────────┘  │     │
-│  │ (9B AR model)   │            │         │          │     │
-│  └─────────────────┘            │         ▼          │     │
-│         ▲                       │  ┌───────────────┐  │     │
-│         │                       │  │  VAE Decode   │──┼──► Image
-│    Text/Image                   │  └───────────────┘  │     │
-│      Input                      └─────────────────────┘     │
-│                                                             │
-└─────────────────────────────────────────────────────────────┘
+Stage 0 (AR Model)                Stage 1 (Diffusion)
+┌───────────────────┐            ┌─────────────────────┐
+│ vLLM-optimized    │  prior     │  GlmImagePipeline   │
+│ GlmImageFor       │──tokens──►│  ┌───────────────┐  │
+│ Conditional       │            │  │ DiT Denoiser  │  │
+│ Generation        │            │  └───────┬───────┘  │
+│ (9B AR model)     │            │          ▼          │
+└───────────────────┘            │  ┌───────────────┐  │
+        ▲                        │  │  VAE Decode   │──┼──► Image
+        │                        │  └───────────────┘  │
+   Text / Image                  └─────────────────────┘
+     Input
 ```
 
-## Features
+## Text-to-Image
 
-- **vLLM-optimized AR**: Uses PagedAttention and tensor parallelism for faster prior token generation
-- **Flexible deployment**: AR and Diffusion stages can run on different GPUs
-- **Text-to-Image**: Generate images from text descriptions
-- **Image-to-Image**: Edit existing images with text prompts
+```python
+from vllm_omni.entrypoints.omni import Omni
 
-## Usage
-
-### Text-to-Image
-
-```bash
-python end2end.py \
-    --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \
-    --prompt "A beautiful sunset over the ocean with sailing boats" \
-    --height 1024 \
-    --width 1024 \
-    --output output_t2i.png
+if __name__ == "__main__":
+    omni = Omni(model="zai-org/GLM-Image")
+    outputs = omni.generate(
+        "A photorealistic mountain landscape at sunset",
+        sampling_params={
+            "height": 1024,
+            "width": 1024,
+            "num_inference_steps": 50,
+            "guidance_scale": 1.5,
+            "seed": 42,
+        },
+    )
+    outputs[0].request_output.images[0].save("output.png")
 ```
 
-### Image-to-Image (Image Editing)
+## Image-to-Image (Image Editing)
 
-```bash
-python end2end.py \
-    --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \
-    --prompt "Transform this scene into a winter wonderland" \
-    --image input.png \
-    --output output_i2i.png
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+if __name__ == "__main__":
+    omni = Omni(model="zai-org/GLM-Image")
+    outputs = omni.generate(
+        {
+            "prompt": "Convert this image to watercolor style",
+            "multi_modal_data": {
+                "image": "input.png",
+            },
+        },
+        sampling_params={
+            "height": 1024,
+            "width": 1024,
+            "num_inference_steps": 50,
+            "guidance_scale": 1.5,
+            "seed": 42,
+        },
+    )
+    outputs[0].request_output.images[0].save("output.png")
 ```
 
-### With Custom Parameters
+## Generation Parameters
 
-```bash
-python end2end.py \
-    --model-path /path/to/glm-image \
-    --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \
-    --prompt "A photorealistic cat sitting on a window sill" \
-    --height 1024 \
-    --width 1024 \
-    --num-inference-steps 50 \
-    --guidance-scale 1.5 \
-    --seed 42 \
-    --output output.png
-```
+| Parameter             | Type  | Default | Description                         |
+| --------------------- | ----- | ------- | ----------------------------------- |
+| `height`              | int   | 1024    | Image height in pixels              |
+| `width`               | int   | 1024    | Image width in pixels               |
+| `num_inference_steps` | int   | 50      | Number of diffusion denoising steps |
+| `guidance_scale`      | float | 1.5     | Classifier-free guidance scale      |
+| `seed`                | int   | None    | Optional random seed                |
+| `negative_prompt`     | str   | None    | Negative prompt                     |
 
-## Shell Scripts
+## VRAM Requirements
 
-### Run Text-to-Image
-
-```bash
-./run_t2i.sh
-```
-
-### Run Image-to-Image
-
-```bash
-./run_i2i.sh --image /path/to/input.png
-```
-
-## Stage Configuration
-
-The stage config (`glm_image.yaml`) defines:
-
-- **Stage 0 (AR)**: Uses `GPUARWorker` with vLLM engine
-
-  - Model: `GlmImageForConditionalGeneration`
-  - Output: `token_ids` (prior tokens)
-
-- **Stage 1 (Diffusion)**: Uses diffusion engine
-  - Model: `GlmImagePipeline`
-  - Output: Generated image
-
-See `vllm_omni/model_executor/stage_configs/glm_image.yaml` for full configuration.
-
-## Comparison with Single-Stage
-
-| Aspect      | Single-Stage (transformers) | Multistage (vLLM)   |
-| ----------- | --------------------------- | ------------------- |
-| AR Model    | transformers native         | vLLM PagedAttention |
-| Memory      | Higher (no KV cache opt)    | Lower (optimized)   |
-| Throughput  | Lower                       | Higher              |
-| Flexibility | Single GPU                  | Multi-GPU support   |
-
-## Troubleshooting
-
-### OOM Error
-
-Try reducing memory usage:
-
-```bash
-# In glm_image.yaml, adjust:
-gpu_memory_utilization: 0.5  # Reduce from 0.6
-```
-
-### Slow Initialization
-
-The first run loads model weights. Subsequent runs are faster:
-
-```bash
---stage-init-timeout 900  # Increase timeout for slow storage
-```
-
-## Requirements
-
-- vLLM-Omni with GLM-Image support
-- CUDA-capable GPU (recommended: H100/A100 with 80GB)
-- GLM-Image model weights
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/glm_image/end2end.py"
-    ``````
-??? abstract "run_i2i.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/glm_image/run_i2i.sh"
-    ``````
-??? abstract "run_t2i.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/glm_image/run_t2i.sh"
-    ``````
+| Stage             | VRAM                   |
+| :---------------- | :--------------------- |
+| Stage-0 (AR)      | **~18 GiB + KV Cache** |
+| Stage-1 (DiT+VAE) | **~20 GiB**            |
+| Total             | **~38 GiB + KV Cache** |

docs/user_guide/examples/offline_inference/internvla_a1.md

@@ -0,0 +1,205 @@
+# InternVLA-A1
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/internvla_a1>.
+
+
+## Setup
+This example is adapted from: https://github.com/InternRobotics/InternVLA-A1/blob/master/tests/policies/internvla_a1_3b/open_loop_genie1_real.ipynb
+
+This example runs the single-path vLLM offline inference workflow for InternVLA-A1 open-loop action prediction.
+
+Before running the script, export the required local paths:
+
+```bash
+# hf Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen
+export INTERNVLA_A1_MODEL_DIR=/path/to/InternVLA-A1-3B-ft-pen
+# hf download InternRobotics/InternData-A1 real_lerobotv30/genie1/Genie1-Place_Markpen.tar.gz --repo-type dataset --local-dir /path/to/Genie1-Place_Markpen
+export INTERNVLA_A1_DATASET_DIR=/path/to/Genie1-Place_Markpen
+export INTERNVLA_A1_PROCESSOR_DIR=/path/to/Qwen3-VL-2B-Instruct
+# hf tenstep/Cosmos-Tokenizer-CI8x8-SafeTensors
+export INTERNVLA_A1_COSMOS_DIR=/path/to/Cosmos-Tokenizer-CI8x8-SafeTensor
+```
+
+The shell entrypoint also accepts these variables as defaults, so you can keep the command line itself short.
+
+`INTERNVLA_A1_COSMOS_DIR` is expected to contain:
+
+- `encoder.safetensors`
+- `decoder.safetensors`
+
+Reference Hugging Face repo: `tenstep/Cosmos-Tokenizer-CI8x8-SafeTensors`
+
+## Run examples
+
+Get into the example folder:
+
+```bash
+cd examples/offline_inference/internvla_a1
+```
+
+### Run one sample
+
+This runs a single sample through the maintained vLLM registry path.
+
+```bash
+bash run.sh --num-samples 1 --num-episodes 0
+```
+
+### Run open-loop evaluation against GT
+
+This runs one episode, writes evaluation metrics, and saves prediction-vs-GT plots when `matplotlib` is available.
+
+```bash
+bash run.sh --num-episodes 1
+```
+
+Outputs are written under:
+
+```bash
+outputs/internvla_a1/vllm_infer/
+```
+
+Typical files:
+
+- `summary.json`: top-level run summary
+- `registry/log.json`: per-episode GT comparison metrics
+- `registry/plots/*.jpg`: prediction-vs-GT figures
+
+### Eager float32 validation
+
+Use eager attention and float32 when you want the most stable numerical comparison baseline.
+
+```bash
+bash run.sh \
+  --num-episodes 1 \
+  --attn-implementation eager \
+  --dtype float32
+```
+
+### Optional runtime switches
+
+- `--dtype {bfloat16,float32}`: choose inference dtype
+- `--attn-implementation {eager,sdpa}`: switch attention backend
+- `--enable-regional-compile`: enable regional `torch.compile`
+- `--enable-warmup`: run pipeline warmup in initialization
+- `--skip-plots`: skip plot generation even if `matplotlib` is installed
+
+### Collect results and performance logs
+
+Use the helper below when you want to collect one-sample output, forward latency, GT comparison, plots, timing logs, and optional pytest output into a single result directory:
+
+```bash
+bash collect_results.sh
+```
+
+The script writes a timestamped directory under:
+
+```bash
+outputs/internvla_a1/collected_results/
+```
+
+Typical result files:
+
+- `env_summary.txt`: environment and path summary
+- `sample_run.log`: one-sample inference log
+- `forward_benchmark/forward_latency.json`: isolated `pipeline.forward` latency summary
+- `eval_run.log`: open-loop GT evaluation log
+- `eval_outputs/summary.json`: top-level output summary
+- `eval_outputs/registry/log.json`: GT comparison metrics
+- `eval_outputs/registry/plots/*.jpg`: prediction-vs-GT figures
+- `*_time.txt`: timing output
+- `*_gpu.csv`: sampled GPU usage
+
+### Benchmark forward latency
+
+Use the dedicated mode below when you want the isolated `pipeline.forward` latency instead of end-to-end script time:
+
+```bash
+python end2end.py \
+  --model-dir "$INTERNVLA_A1_MODEL_DIR" \
+  --dataset-dir "$INTERNVLA_A1_DATASET_DIR" \
+  --benchmark-forward \
+  --dtype bfloat16 \
+  --attn-implementation eager \
+  --warmup-iters 3 \
+  --benchmark-iters 10 \
+  --output-dir outputs/internvla_a1/forward_benchmark
+```
+
+The output JSON is written to `outputs/internvla_a1/forward_benchmark/forward_latency.json` and contains:
+
+- `cold_start_ms`
+- `warmup_summary`
+- `benchmark_summary`
+- `benchmark_samples_ms`
+
+### Reference results
+
+Reference run collected on `1x NVIDIA H200`, `bfloat16`, `eager`:
+
+- one-sample end-to-end run: `38s`
+- one-episode GT evaluation run: `45s`
+- `average_mse = 1.7173260857816786e-05`
+- `average_mae = 0.0011860118247568607`
+- `average_mse_joint = 7.42028441891307e-06`
+- `average_mae_joint = 0.0010777723509818316`
+- `average_mse_gripper = 8.544408774469048e-05`
+- `average_mae_gripper = 0.0019436875591054559`
+
+The reference plot filename is:
+
+- `eval_outputs/registry/plots/vllm_registry_open_loop_ep0.jpg`
+
+## Measure latency and VRAM
+
+You can collect end-to-end runtime and peak host memory with:
+
+```bash
+/usr/bin/time -v bash run.sh --num-episodes 1
+```
+
+For GPU memory snapshots, use:
+
+```bash
+nvidia-smi --query-gpu=name,memory.total,memory.used --format=csv
+```
+
+Run `nvidia-smi` before and during inference to record the peak device memory for your PR report.
+
+## FAQ
+
+If `matplotlib` is missing, evaluation still runs and only plot generation is skipped. To enable plots:
+
+```bash
+pip install matplotlib
+```
+
+## Example materials
+
+- `end2end.py`: main offline inference and GT evaluation entrypoint
+- `run.sh`: shell wrapper with local path env vars
+- `collect_results.sh`: helper to gather result summaries and performance logs into one directory
+- `internvla_a1_common.py`: dataset, evaluation, and plotting helpers
+
+## Embedded source listings
+
+??? abstract "collect_results.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/internvla_a1/collect_results.sh"
+    ``````
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/internvla_a1/end2end.py"
+    ``````
+??? abstract "internvla_a1_common.py"
+    ``````py
+    --8<-- "examples/offline_inference/internvla_a1/internvla_a1_common.py"
+    ``````
+??? abstract "run.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/internvla_a1/run.sh"
+    ``````
+??? abstract "standalone/outputs/qwena1/standalone_infer/log.json"
+    ``````json
+    --8<-- "examples/offline_inference/internvla_a1/standalone/outputs/qwena1/standalone_infer/log.json"
+    ``````

docs/user_guide/examples/online_serving/glm_image.md

@@ -1,99 +1,96 @@
 # GLM-Image Online Serving
 
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/glm_image>.
+GLM-Image is a 2-stage image generation model (AR + Diffusion) supported by vLLM-Omni's
+declarative config system. The pipeline topology and stage structure are declared in
+`vllm_omni/model_executor/models/glm_image/pipeline.py`; deployment knobs (GPU placement,
+memory, sampling params) live in `vllm_omni/deploy/glm_image.yaml`.
 
-
-This example demonstrates how to deploy GLM-Image for online image generation using vLLM-Omni.
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Run examples (GLM-Image)
-
-**Note**: These examples work with the default configuration on **2× NVIDIA A100 (80GB)** or equivalent. Stage 0 (AR) and Stage 1 (Diffusion) each use one GPU by default. For single-GPU setups, modify the stage configuration to share the same device.
-
-### Launch the Server
+## Start Server
 
 ```bash
-# Use default configuration
 vllm serve zai-org/GLM-Image --omni --port 8091
 ```
 
-Or use the convenience script:
+The config system auto-detects the pipeline from the model's `model_index.json` — no
+manual `--stage-configs-path` or `--deploy-config` needed.
+
+By default, stage 0 (AR) runs on GPU 0 and stage 1 (Diffusion) on GPU 1. To colocate
+both stages on a single GPU, override per stage:
 
 ```bash
-cd examples/online_serving/glm_image
-bash run_server.sh
+vllm serve zai-org/GLM-Image --omni --port 8091 \
+    --stage-0-devices 0 --stage-1-devices 0
 ```
 
-If you have a custom stage configs file:
-
-```bash
-vllm serve zai-org/GLM-Image --omni --port 8091 --stage-configs-path /path/to/glm_image.yaml
-```
-
-### Send Requests
-
-Get into the glm_image folder:
-
-```bash
-cd examples/online_serving/glm_image
-```
-
-Send request via Python:
-
-```bash
-python openai_chat_client.py --prompt "A cute cat sitting on a window sill"
-```
-
-The Python client supports the following command-line arguments:
-
-- `--prompt` (or `-p`): Text prompt for generation (default: `A beautiful sunset over the ocean with sailing boats`)
-- `--output` (or `-o`): Output file path (default: `glm_image_output.png`)
-- `--server` (or `-s`): Server URL (default: `http://localhost:8091`)
-- `--image` (or `-i`): Input image path (for image-to-image editing)
-- `--height`: Image height in pixels (default: 1024)
-- `--width`: Image width in pixels (default: 1024)
-- `--steps`: Number of inference steps (default: 50)
-- `--guidance-scale`: Classifier-free guidance scale (default: 1.5)
-- `--seed`: Random seed (default: 42)
-- `--negative`: Negative prompt
-
-## Modality Control
-
-GLM-Image supports **text-to-image** and **image-to-image** modes.
-
-The default yaml configuration deploys AR on GPU 0 and DiT on GPU 1. You can use the default configuration file: [`glm_image.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/model_executor/stage_configs/glm_image.yaml)
-
-| Mode           | Input        | Output | Description                        |
-| -------------- | ------------ | ------ | ---------------------------------- |
-| Text-to-Image  | Text         | Image  | Generate images from text prompts  |
-| Image-to-Image | Image + Text | Image  | Edit images with text instructions |
+## API Calls
 
 ### Text-to-Image
 
 ```bash
-python openai_chat_client.py \
-    --prompt "A photorealistic mountain landscape at sunset" \
-    --height 1024 \
-    --width 1024 \
-    --output landscape.png
-
-# Or use the curl script:
-bash run_curl_text_to_image.sh "A futuristic city skyline at night"
+curl -s http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      {"role": "user", "content": "A photorealistic mountain landscape at sunset"}
+    ],
+    "extra_body": {
+      "height": 1024,
+      "width": 1024,
+      "num_inference_steps": 50,
+      "guidance_scale": 1.5,
+      "seed": 42
+    }
+  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png
 ```
 
 ### Image-to-Image (Image Editing)
 
 ```bash
-python openai_chat_client.py \
-    --prompt "Convert this image to watercolor style" \
-    --image input.png \
-    --output watercolor.png
+curl -s http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      {
+        "role": "user",
+        "content": [
+          {"type": "text", "text": "Convert this image to watercolor style"},
+          {"type": "image_url", "image_url": {"url": "data:image/png;base64,$(base64 -w0 input.png)}"}
+        ]
+      }
+    ],
+    "extra_body": {
+      "height": 1024,
+      "width": 1024,
+      "num_inference_steps": 50,
+      "guidance_scale": 1.5
+    }
+  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png
+```
 
-# Or use the curl script:
-bash run_curl_image_edit.sh input.png "Convert to watercolor style"
+### Using the OpenAI Python SDK
+
+```python
+from openai import OpenAI
+import base64
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+
+response = client.chat.completions.create(
+    model="zai-org/GLM-Image",
+    messages=[{"role": "user", "content": "A beautiful sunset over the ocean"}],
+    extra_body={
+        "height": 1024,
+        "width": 1024,
+        "num_inference_steps": 50,
+        "guidance_scale": 1.5,
+        "seed": 42,
+    },
+)
+
+img_url = response.choices[0].message.content[0].image_url.url
+_, b64_data = img_url.split(",", 1)
+with open("output.png", "wb") as f:
+    f.write(base64.b64decode(b64_data))
 ```
 
 For general-purpose request methods (curl, OpenAI SDK, Python `requests`), see
@@ -116,7 +113,7 @@ dimensions and count, use `size` and `n` rather than `height` or `width`.
 | `width`               | int   | 1024    | Image width in pixels               |
 | `num_inference_steps` | int   | 50      | Number of diffusion denoising steps |
 | `guidance_scale`      | float | 1.5     | Classifier-free guidance scale      |
-| `seed`                | int   | None    | Optional random seed; `/v1/images/*` generates one server-side if omitted |
+| `seed`                | int   | None    | Optional random seed                |
 | `negative_prompt`     | str   | None    | Negative prompt                     |
 
 ## Response Format
@@ -150,13 +147,12 @@ dimensions and count, use `size` and `n` rather than `height` or `width`.
 ## Extract Image
 
 ```bash
-# From a saved JSON response
 cat response.json | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png
 ```
 
 ## Architecture
 
-GLM-Image uses a 2-stage multistage pipeline:
+GLM-Image uses a 2-stage pipeline:
 
 ```
 Stage 0 (AR Model)                Stage 1 (Diffusion)
@@ -181,41 +177,13 @@ Stage 0 (AR Model)                Stage 1 (Diffusion)
 | Stage-1 (DiT+VAE) | **~20 GiB**            |
 | Total             | **~38 GiB + KV Cache** |
 
-## File Description
-
-| File                        | Description                           |
-| --------------------------- | ------------------------------------- |
-| `run_server.sh`             | Server startup script                 |
-| `run_curl_text_to_image.sh` | Text-to-image curl example            |
-| `run_curl_image_edit.sh`    | Image-to-image (editing) curl example |
-| `openai_chat_client.py`     | Python client (t2i + i2i)             |
-
 ## FAQ
 
-- If you encounter OOM errors, adjust `gpu_memory_utilization` in the stage config:
+- If you encounter OOM errors, adjust `gpu_memory_utilization` in the deploy config:
 
 ```yaml
-# In glm_image.yaml, reduce from default 0.6:
+# In vllm_omni/deploy/glm_image.yaml, reduce from default 0.6:
 gpu_memory_utilization: 0.5
 ```
 
 - The first request may be slow due to model warmup. Subsequent requests will be faster.
-
-## Example materials
-
-??? abstract "openai_chat_client.py"
-    ``````py
-    --8<-- "examples/online_serving/glm_image/openai_chat_client.py"
-    ``````
-??? abstract "run_curl_image_edit.sh"
-    ``````sh
-    --8<-- "examples/online_serving/glm_image/run_curl_image_edit.sh"
-    ``````
-??? abstract "run_curl_text_to_image.sh"
-    ``````sh
-    --8<-- "examples/online_serving/glm_image/run_curl_text_to_image.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/glm_image/run_server.sh"
-    ``````

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -28,11 +28,13 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
 ### Launch individual stages (stage-based CLI)
 
 Adopt the stage-based CLI architecture to independently instantiate execution processes per functional stage.
+The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via
+`CUDA_VISIBLE_DEVICES`.
 
 **1. Stage 0 (Thinker + API server)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
@@ -42,7 +44,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 **2. Stage 1 (Talker)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -52,7 +54,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 **3. Stage 2 (Code2Wav)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 2 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -75,7 +77,7 @@ that kind of change. Since each command launches one stage, just pass the knob
 directly on that stage command:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \

examples/offline_inference/internvla_a1/README.md

@@ -0,0 +1,30 @@
+# InternVLA-A1
+
+Full usage and result-reporting guidance lives in [docs/user_guide/examples/offline_inference/internvla_a1.md](../../../docs/user_guide/examples/offline_inference/internvla_a1.md).
+
+Quick start:
+
+```bash
+export INTERNVLA_A1_MODEL_DIR=/path/to/InternVLA-A1-3B-ft-pen
+export INTERNVLA_A1_DATASET_DIR=/path/to/Genie1-Place_Markpen
+export INTERNVLA_A1_PROCESSOR_DIR=/path/to/Qwen3-VL-2B-Instruct
+# hf tenstep/Cosmos-Tokenizer-CI8x8-SafeTensors
+export INTERNVLA_A1_COSMOS_DIR=/path/to/Cosmos-Tokenizer-CI8x8-SafeTensor
+
+bash run.sh --num-samples 1 --num-episodes 0
+bash run.sh --num-episodes 1
+bash collect_results.sh
+```
+
+Expected files under `INTERNVLA_A1_COSMOS_DIR`:
+
+- `encoder.safetensors`
+- `decoder.safetensors`
+
+Reference Hugging Face repo: `tenstep/Cosmos-Tokenizer-CI8x8-SafeTensors`
+
+Key entrypoints:
+
+- `run.sh`: wrapper for offline inference and GT evaluation
+- `collect_results.sh`: collect sample output, latency, metrics, plots, and logs
+- `end2end.py`: underlying Python entrypoint

examples/offline_inference/internvla_a1/collect_results.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$ROOT_DIR/../../.." && pwd)"
+TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
+RESULT_ROOT="${INTERNVLA_A1_RESULT_DIR:-$REPO_ROOT/outputs/internvla_a1/collected_results/$TIMESTAMP}"
+
+: "${INTERNVLA_A1_MODEL_DIR:?Please export INTERNVLA_A1_MODEL_DIR=/path/to/InternVLA-A1-3B-ft-pen}"
+: "${INTERNVLA_A1_DATASET_DIR:?Please export INTERNVLA_A1_DATASET_DIR=/path/to/Genie1-Place_Markpen}"
+: "${INTERNVLA_A1_PROCESSOR_DIR:?Please export INTERNVLA_A1_PROCESSOR_DIR=/path/to/Qwen3-VL-2B-Instruct}"
+: "${INTERNVLA_A1_COSMOS_DIR:?Please export INTERNVLA_A1_COSMOS_DIR=/path/to/Cosmos-Tokenizer-CI8x8-SafeTensor}"
+
+EVAL_OUTPUT_DIR="$RESULT_ROOT/eval_outputs"
+GPU_MONITOR_INTERVAL="${GPU_MONITOR_INTERVAL:-1}"
+
+mkdir -p "$RESULT_ROOT"
+
+export INTERNVLA_A1_MODEL_DIR
+export INTERNVLA_A1_DATASET_DIR
+export INTERNVLA_A1_PROCESSOR_DIR
+export INTERNVLA_A1_COSMOS_DIR
+
+write_env_summary() {
+  cat >"$RESULT_ROOT/env_summary.txt" <<EOF
+timestamp=$TIMESTAMP
+repo_root=$REPO_ROOT
+result_root=$RESULT_ROOT
+python=$(command -v python || true)
+python_version=$(python --version 2>&1 || true)
+model_dir=$INTERNVLA_A1_MODEL_DIR
+dataset_dir=$INTERNVLA_A1_DATASET_DIR
+processor_dir=$INTERNVLA_A1_PROCESSOR_DIR
+cosmos_dir=$INTERNVLA_A1_COSMOS_DIR
+pwd=$(pwd)
+EOF
+}
+
+capture_gpu_snapshot() {
+  local output_file="$1"
+  if command -v nvidia-smi >/dev/null 2>&1; then
+    nvidia-smi --query-gpu=index,name,memory.total,memory.used,utilization.gpu --format=csv >"$output_file" 2>&1 || true
+  else
+    echo "nvidia-smi not found" >"$output_file"
+  fi
+}
+
+start_gpu_monitor() {
+  local output_file="$1"
+  if ! command -v nvidia-smi >/dev/null 2>&1; then
+    echo "nvidia-smi not found" >"$output_file"
+    return 1
+  fi
+
+  (
+    echo "timestamp,index,name,memory.total [MiB],memory.used [MiB],utilization.gpu [%]"
+    while true; do
+      local now
+      now="$(date '+%Y-%m-%d %H:%M:%S')"
+      nvidia-smi --query-gpu=index,name,memory.total,memory.used,utilization.gpu --format=csv,noheader,nounits \
+        | awk -F',' -v ts="$now" '{gsub(/^[ \t]+|[ \t]+$/, "", $0); print ts "," $0}'
+      sleep "$GPU_MONITOR_INTERVAL"
+    done
+  ) >"$output_file" 2>/dev/null &
+  echo $!
+}
+
+stop_gpu_monitor() {
+  local monitor_pid="${1:-}"
+  if [[ -n "$monitor_pid" ]] && kill -0 "$monitor_pid" 2>/dev/null; then
+    kill "$monitor_pid" 2>/dev/null || true
+    wait "$monitor_pid" 2>/dev/null || true
+  fi
+}
+
+copy_eval_outputs() {
+  local source_dir="$1"
+  local target_dir="$2"
+  local source_real=""
+  local target_real=""
+  source_real="$(realpath "$source_dir" 2>/dev/null || echo "$source_dir")"
+  target_real="$(realpath "$target_dir" 2>/dev/null || echo "$target_dir")"
+  if [[ "$source_real" == "$target_real" ]]; then
+    return 0
+  fi
+  mkdir -p "$target_dir"
+  if [[ -d "$source_dir" ]]; then
+    cp -r "$source_dir"/. "$target_dir"/
+  fi
+}
+
+run_with_artifacts() {
+  local name="$1"
+  shift
+
+  local log_file="$RESULT_ROOT/${name}.log"
+  local time_file="$RESULT_ROOT/${name}_time.txt"
+  local gpu_file="$RESULT_ROOT/${name}_gpu.csv"
+  local status_file="$RESULT_ROOT/${name}_status.txt"
+  local monitor_pid=""
+
+  monitor_pid="$(start_gpu_monitor "$gpu_file" || true)"
+  set +e
+  local exit_code=0
+  local start_ts=0
+  local end_ts=0
+  start_ts="$(date +%s)"
+  if [[ -x /usr/bin/time ]]; then
+    /usr/bin/time -v -o "$time_file" "$@" >"$log_file" 2>&1
+    exit_code=$?
+    if [[ ! -s "$time_file" ]]; then
+      end_ts="$(date +%s)"
+      {
+        echo "timing_mode=usr_bin_time_empty_fallback"
+        echo "start_ts=$start_ts"
+        echo "end_ts=$end_ts"
+        echo "elapsed_seconds=$((end_ts - start_ts))"
+      } >"$time_file"
+    fi
+  else
+    "$@" >"$log_file" 2>&1
+    exit_code=$?
+    end_ts="$(date +%s)"
+    {
+      echo "timing_mode=shell_date"
+      echo "start_ts=$start_ts"
+      echo "end_ts=$end_ts"
+      echo "elapsed_seconds=$((end_ts - start_ts))"
+    } >"$time_file"
+  fi
+  set -e
+  stop_gpu_monitor "$monitor_pid"
+
+  echo "exit_code=$exit_code" >"$status_file"
+  if [[ $exit_code -ne 0 ]]; then
+    echo "[error] command failed for $name, see $log_file"
+    return $exit_code
+  fi
+}
+
+write_manifest() {
+  cat >"$RESULT_ROOT/README.txt" <<EOF
+InternVLA-A1 collected results
+
+Key files:
+- env_summary.txt: environment and path summary
+- sample_run.log / sample_run_time.txt: one-sample functional run
+- forward_benchmark.log / forward_benchmark_time.txt: pure pipeline.forward latency benchmark
+- eval_run.log / eval_run_time.txt: GT evaluation run
+- pytest_e2e.log / pytest_e2e_time.txt: offline e2e pytest result
+- gpu_info_before.csv / gpu_info_after.csv: point-in-time GPU snapshots
+- *_gpu.csv: sampled GPU usage during each run
+- eval_outputs/: copied output directory from the GT evaluation run
+
+Important outputs:
+- forward_benchmark/forward_latency.json
+- eval_outputs/summary.json
+- eval_outputs/registry/log.json
+- eval_outputs/registry/plots/
+EOF
+}
+
+write_skip_artifact() {
+  local name="$1"
+  local reason="$2"
+  echo "$reason" >"$RESULT_ROOT/${name}.log"
+  echo "timing_mode=skipped" >"$RESULT_ROOT/${name}_time.txt"
+  echo "exit_code=0" >"$RESULT_ROOT/${name}_status.txt"
+  echo "skipped_reason=$reason" >>"$RESULT_ROOT/${name}_status.txt"
+  if [[ ! -f "$RESULT_ROOT/${name}_gpu.csv" ]]; then
+    echo "skipped,$reason" >"$RESULT_ROOT/${name}_gpu.csv"
+  fi
+}
+
+write_env_summary
+write_manifest
+capture_gpu_snapshot "$RESULT_ROOT/gpu_info_before.csv"
+
+run_with_artifacts \
+  "sample_run" \
+  bash "$ROOT_DIR/run.sh" \
+  --output-dir "$RESULT_ROOT/sample_outputs" \
+  --num-samples 1 \
+  --num-episodes 0
+
+run_with_artifacts \
+  "forward_benchmark" \
+  python "$ROOT_DIR/end2end.py" \
+  --model-dir "$INTERNVLA_A1_MODEL_DIR" \
+  --dataset-dir "$INTERNVLA_A1_DATASET_DIR" \
+  --benchmark-forward \
+  --dtype bfloat16 \
+  --attn-implementation eager \
+  --warmup-iters 3 \
+  --benchmark-iters 10 \
+  --output-dir "$RESULT_ROOT/forward_benchmark"
+
+run_with_artifacts \
+  "eval_run" \
+  bash "$ROOT_DIR/run.sh" \
+  --output-dir "$EVAL_OUTPUT_DIR" \
+  --num-episodes 1
+
+copy_eval_outputs "$EVAL_OUTPUT_DIR" "$RESULT_ROOT/eval_outputs"
+
+if python - <<'PY' >/dev/null 2>&1
+import importlib.util
+raise SystemExit(0 if importlib.util.find_spec("pytest") else 1)
+PY
+then
+  run_with_artifacts \
+    "pytest_e2e" \
+    python -m pytest -sv tests/e2e/offline_inference/test_internvla_a1.py -m advanced_model
+else
+  write_skip_artifact "pytest_e2e" "pytest is not installed in the current python environment"
+fi
+
+capture_gpu_snapshot "$RESULT_ROOT/gpu_info_after.csv"
+
+echo "Results written to: $RESULT_ROOT"

examples/offline_inference/internvla_a1/end2end.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics
+import sys
+import time
+from pathlib import Path
+
+import torch
+
+_WORKSPACE_ROOT = Path(__file__).resolve().parent
+_REPO_ROOT = _WORKSPACE_ROOT.parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    # Keep the shared helper import clean by doing script-only path bootstrap here.
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from internvla_a1_common import (  # noqa: E402
+    A2DOpenLoopDataset,
+    collate_open_loop_samples,
+    make_shared_noise,
+    run_open_loop_evaluation,
+    select_indices,
+    tensor_dtype,
+    tensor_sha256,
+)
+
+from vllm_omni.diffusion.data import OmniDiffusionConfig  # noqa: E402
+from vllm_omni.diffusion.models.internvla_a1 import (  # noqa: E402
+    InternVLAA1Config,
+    InternVLAA1TrainMetadata,
+)
+from vllm_omni.diffusion.models.internvla_a1.config import OBS_STATE  # noqa: E402
+from vllm_omni.diffusion.registry import initialize_model  # noqa: E402
+from vllm_omni.diffusion.request import OmniDiffusionRequest  # noqa: E402
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams  # noqa: E402
+
+
+def _required_path_arg(env_name: str, cli_value: str | None) -> str:
+    value = cli_value or os.getenv(env_name)
+    if not value:
+        raise ValueError(f"Missing required path: set --{env_name.lower().replace('_', '-')} or {env_name}.")
+    return value
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run vLLM InternVLA-A1 inference on a few samples.")
+    parser.add_argument("--model-dir")
+    parser.add_argument("--dataset-dir")
+    parser.add_argument("--num-samples", type=int, default=1)
+    parser.add_argument("--num-episodes", type=int, default=0)
+    parser.add_argument("--seed", type=int, default=1234)
+    parser.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--compile-model", action="store_true")
+    parser.add_argument("--attn-implementation", default="eager")
+    parser.add_argument("--enable-regional-compile", action="store_true")
+    parser.add_argument("--enable-warmup", action="store_true")
+    parser.add_argument("--strict-load", action="store_true")
+    parser.add_argument("--output-dir", default="outputs/internvla_a1/vllm_infer")
+    parser.add_argument("--skip-plots", action="store_true")
+    parser.add_argument("--benchmark-forward", action="store_true")
+    parser.add_argument("--warmup-iters", type=int, default=3)
+    parser.add_argument("--benchmark-iters", type=int, default=10)
+    return parser.parse_args()
+
+
+def build_od_config(args: argparse.Namespace, processor_model_name: str) -> OmniDiffusionConfig:
+    return OmniDiffusionConfig(
+        model=str(Path(args.model_dir).resolve()),
+        model_class_name="InternVLAA1Pipeline",
+        dtype=tensor_dtype(args.dtype),
+        custom_pipeline_args={
+            "device": args.device,
+            "dtype": args.dtype,
+            "compile_model": args.compile_model,
+            "attn_implementation": args.attn_implementation,
+            "enable_regional_compile": args.enable_regional_compile,
+            "enable_warmup": args.enable_warmup,
+            "strict_load": args.strict_load,
+            "processor_model_name": processor_model_name,
+        },
+    )
+
+
+def build_dataset(args: argparse.Namespace) -> tuple[A2DOpenLoopDataset, InternVLAA1Config, InternVLAA1TrainMetadata]:
+    model_dir = Path(args.model_dir)
+    config = InternVLAA1Config.from_pretrained(model_dir)
+    config.device = args.device
+    config.dtype = args.dtype
+    config.compile_model = args.compile_model
+    config.attn_implementation = args.attn_implementation
+    config.enable_regional_compile = args.enable_regional_compile
+
+    train_meta = InternVLAA1TrainMetadata.from_pretrained(model_dir)
+    with open(model_dir / "stats.json", encoding="utf-8") as f:
+        train_stats = json.load(f)["a2d"]
+
+    dataset = A2DOpenLoopDataset(
+        args.dataset_dir,
+        config=config,
+        train_stats=train_stats,
+    )
+    return dataset, config, train_meta
+
+
+def run_one_path(
+    pipeline,
+    dataset: A2DOpenLoopDataset,
+    config: InternVLAA1Config,
+    args: argparse.Namespace,
+    indices: list[int],
+) -> list[dict[str, object]]:
+    results: list[dict[str, object]] = []
+    for index in indices:
+        sample = dataset.get_sample(index)
+        batch_inputs, _ = collate_open_loop_samples([sample], device=args.device, dtype=tensor_dtype(args.dtype))
+        noise = make_shared_noise(
+            args.seed,
+            index,
+            (
+                batch_inputs[OBS_STATE].shape[0],
+                config.chunk_size,
+                config.max_action_dim,
+            ),
+            args.device,
+        )
+        pred = run_pipeline_forward(pipeline, batch_inputs, noise)
+        pred = pred[:, :, : dataset.physical_action_dim].to(torch.float32).cpu()
+        results.append(
+            {
+                "path": "registry",
+                "index": index,
+                "episode_index": sample.episode_index,
+                "task": sample.task,
+                "seed": args.seed,
+                "shape": list(pred.shape),
+                "mean": float(pred.mean().item()),
+                "std": float(pred.std().item()),
+                "action_sha256": tensor_sha256(pred),
+                "first_action_prefix": pred[0, 0, :8].tolist(),
+            }
+        )
+    return results
+
+
+def run_pipeline_forward(
+    pipeline,
+    batch_inputs: dict[str, torch.Tensor],
+    noise: torch.Tensor,
+) -> torch.Tensor:
+    output = pipeline.forward(
+        OmniDiffusionRequest(
+            prompts=[""],
+            sampling_params=OmniDiffusionSamplingParams(
+                extra_args={
+                    "batch_inputs": batch_inputs,
+                    "noise": noise,
+                    "decode_image": False,
+                }
+            ),
+        )
+    )
+    if output.error:
+        raise RuntimeError(output.error)
+    if output.output is None:
+        raise RuntimeError("InternVLAA1Pipeline.forward returned no output tensor.")
+    return output.output
+
+
+def _synchronize(device: str) -> None:
+    if device.startswith("cuda") and torch.cuda.is_available():
+        torch.cuda.synchronize()
+
+
+def _latency_summary(values_ms: list[float]) -> dict[str, float]:
+    sorted_values = sorted(values_ms)
+    p50_index = min(len(sorted_values) - 1, round(0.50 * (len(sorted_values) - 1)))
+    p90_index = min(len(sorted_values) - 1, round(0.90 * (len(sorted_values) - 1)))
+    return {
+        "mean_ms": float(statistics.mean(sorted_values)),
+        "stdev_ms": float(statistics.pstdev(sorted_values)) if len(sorted_values) > 1 else 0.0,
+        "min_ms": float(sorted_values[0]),
+        "max_ms": float(sorted_values[-1]),
+        "p50_ms": float(sorted_values[p50_index]),
+        "p90_ms": float(sorted_values[p90_index]),
+    }
+
+
+def benchmark_forward(
+    pipeline,
+    dataset: A2DOpenLoopDataset,
+    config: InternVLAA1Config,
+    args: argparse.Namespace,
+    index: int,
+    output_dir: Path,
+) -> dict[str, object]:
+    sample = dataset.get_sample(index)
+    batch_inputs, _ = collate_open_loop_samples([sample], device=args.device, dtype=tensor_dtype(args.dtype))
+    noise = make_shared_noise(
+        args.seed,
+        index,
+        (
+            batch_inputs[OBS_STATE].shape[0],
+            config.chunk_size,
+            config.max_action_dim,
+        ),
+        args.device,
+    )
+
+    _synchronize(args.device)
+    cold_start_begin = time.perf_counter()
+    pred = run_pipeline_forward(pipeline, batch_inputs, noise)
+    _synchronize(args.device)
+    cold_start_ms = (time.perf_counter() - cold_start_begin) * 1000.0
+
+    warmup_ms: list[float] = []
+    for _ in range(args.warmup_iters):
+        _synchronize(args.device)
+        begin = time.perf_counter()
+        _ = run_pipeline_forward(pipeline, batch_inputs, noise)
+        _synchronize(args.device)
+        warmup_ms.append((time.perf_counter() - begin) * 1000.0)
+
+    benchmark_ms: list[float] = []
+    for _ in range(args.benchmark_iters):
+        _synchronize(args.device)
+        begin = time.perf_counter()
+        _ = run_pipeline_forward(pipeline, batch_inputs, noise)
+        _synchronize(args.device)
+        benchmark_ms.append((time.perf_counter() - begin) * 1000.0)
+
+    pred = pred[:, :, : dataset.physical_action_dim].to(torch.float32).cpu()
+    summary = {
+        "mode": "forward_latency",
+        "model_dir": str(Path(args.model_dir).resolve()),
+        "dataset_dir": str(Path(args.dataset_dir).resolve()),
+        "sample_index": index,
+        "episode_index": sample.episode_index,
+        "task": sample.task,
+        "device": args.device,
+        "dtype": args.dtype,
+        "attn_implementation": args.attn_implementation,
+        "enable_regional_compile": args.enable_regional_compile,
+        "warmup_iters": args.warmup_iters,
+        "benchmark_iters": args.benchmark_iters,
+        "output_shape": list(pred.shape),
+        "cold_start_ms": cold_start_ms,
+        "warmup_summary": _latency_summary(warmup_ms) if warmup_ms else {},
+        "benchmark_summary": _latency_summary(benchmark_ms) if benchmark_ms else {},
+        "benchmark_samples_ms": benchmark_ms,
+    }
+    with open(output_dir / "forward_latency.json", "w", encoding="utf-8") as f:
+        json.dump(summary, f, ensure_ascii=False, indent=2)
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return summary
+
+
+def main() -> None:
+    args = parse_args()
+    args.model_dir = _required_path_arg("INTERNVLA_A1_MODEL_DIR", args.model_dir)
+    args.dataset_dir = _required_path_arg("INTERNVLA_A1_DATASET_DIR", args.dataset_dir)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    dataset, config, train_meta = build_dataset(args)
+    indices = select_indices(dataset, args.num_samples)
+    od_config = build_od_config(args, train_meta.processor_model_name)
+
+    eval_summaries: dict[str, object] = {}
+    pipeline = initialize_model(od_config)
+    if args.benchmark_forward:
+        benchmark_forward(
+            pipeline=pipeline,
+            dataset=dataset,
+            config=config,
+            args=args,
+            index=indices[0],
+            output_dir=output_dir,
+        )
+        return
+    results = run_one_path(
+        pipeline=pipeline,
+        dataset=dataset,
+        config=config,
+        args=args,
+        indices=indices,
+    )
+    if args.num_episodes > 0:
+        eval_summaries["registry"] = run_open_loop_evaluation(
+            mode="vllm_registry",
+            policy=pipeline,
+            config=config,
+            dataset=dataset,
+            train_meta=train_meta,
+            collate_samples=collate_open_loop_samples,
+            run_sample_actions=lambda policy, batch_inputs, noise: run_pipeline_forward(policy, batch_inputs, noise),
+            num_episodes=args.num_episodes,
+            seed=args.seed,
+            device=args.device,
+            dtype=tensor_dtype(args.dtype),
+            output_dir=output_dir / "registry",
+            skip_plots=args.skip_plots,
+        )
+
+    summary = {
+        "mode": "registry",
+        "model_dir": str(Path(args.model_dir).resolve()),
+        "dataset_dir": str(Path(args.dataset_dir).resolve()),
+        "device": args.device,
+        "dtype": args.dtype,
+        "attn_implementation": args.attn_implementation,
+        "enable_regional_compile": args.enable_regional_compile,
+        "seed": args.seed,
+        "indices": indices,
+        "results": results,
+        "output_dir": str(output_dir.resolve()),
+        "eval_summaries": eval_summaries,
+    }
+    with open(output_dir / "summary.json", "w", encoding="utf-8") as f:
+        json.dump(summary, f, ensure_ascii=False, indent=2)
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/internvla_a1/internvla_a1_common.py

@@ -0,0 +1,471 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pyarrow.parquet as pq
+import torch
+import torch.nn.functional as F
+import torchvision
+
+from vllm_omni.diffusion.models.internvla_a1.config import (  # noqa: E402
+    OBS_IMAGES,
+    OBS_STATE,
+    OBS_TASK,
+    InternVLAA1Config,
+)
+
+
+def tensor_dtype(name: str) -> torch.dtype:
+    return torch.bfloat16 if name == "bfloat16" else torch.float32
+
+
+def select_indices(dataset, num_samples: int) -> list[int]:
+    indices: list[int] = []
+    for _, episode_indices in dataset.episode_start_indices(max_episodes=num_samples):
+        if episode_indices:
+            indices.append(episode_indices[0])
+    return indices[:num_samples]
+
+
+def make_shared_noise(seed: int, sample_index: int, shape: tuple[int, ...], device: str) -> torch.Tensor:
+    generator = torch.Generator(device="cpu")
+    generator.manual_seed(seed + sample_index)
+    noise = torch.randn(shape, generator=generator, dtype=torch.float32)
+    return noise.to(device=device)
+
+
+def tensor_sha256(tensor: torch.Tensor) -> str:
+    array = tensor.detach().contiguous().cpu().numpy()
+    return hashlib.sha256(array.tobytes()).hexdigest()
+
+
+def _clamp_index(index: int, start: int, end: int) -> int:
+    return max(start, min(end - 1, index))
+
+
+def _load_parquet_rows(path: Path) -> list[dict[str, Any]]:
+    return pq.read_table(path).to_pylist()
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _stack_stats(stats: dict[str, Any], keys: list[str]) -> dict[str, torch.Tensor]:
+    result = {}
+    for stat_name in ("mean", "std"):
+        values = []
+        for key in keys:
+            values.extend(stats[key][stat_name])
+        result[stat_name] = torch.tensor(values, dtype=torch.float32)
+    return result
+
+
+def normalize_vector(values: torch.Tensor, stats: dict[str, torch.Tensor]) -> torch.Tensor:
+    denom = torch.where(stats["std"] == 0, torch.ones_like(stats["std"]), stats["std"])
+    return (values - stats["mean"]) / denom
+
+
+def unnormalize_vector(values: torch.Tensor, stats: dict[str, torch.Tensor]) -> torch.Tensor:
+    return values * stats["std"] + stats["mean"]
+
+
+class TorchvisionVideoReaderCache:
+    def __init__(self, backend: str = "pyav") -> None:
+        self.backend = backend
+        self._readers: dict[str, Any] = {}
+        torchvision.set_video_backend(backend)
+
+    def get(self, path: str) -> Any:
+        reader = self._readers.get(path)
+        if reader is None:
+            reader = torchvision.io.VideoReader(path, "video")
+            self._readers[path] = reader
+        return reader
+
+    def decode_frames(self, path: str, timestamps: list[float], tolerance_s: float = 1e-4) -> torch.Tensor:
+        reader = self.get(path)
+        first_ts = min(timestamps)
+        last_ts = max(timestamps)
+        reader.seek(first_ts, keyframes_only=self.backend == "pyav")
+
+        loaded_frames: list[torch.Tensor] = []
+        loaded_ts: list[float] = []
+        for frame in reader:
+            current_ts = float(frame["pts"])
+            loaded_frames.append(frame["data"])
+            loaded_ts.append(current_ts)
+            if current_ts >= last_ts:
+                break
+
+        query_ts = torch.tensor(timestamps, dtype=torch.float32)
+        loaded_ts_tensor = torch.tensor(loaded_ts, dtype=torch.float32)
+        distances = torch.cdist(query_ts[:, None], loaded_ts_tensor[:, None], p=1)
+        min_dist, argmin = distances.min(dim=1)
+        if not torch.all(min_dist < tolerance_s):
+            raise RuntimeError(
+                f"Video timestamps are outside tolerance: query={query_ts.tolist()} "
+                f"loaded={loaded_ts_tensor.tolist()} path={path}"
+            )
+        return torch.stack([loaded_frames[i] for i in argmin]).float() / 255.0
+
+
+@dataclass
+class A2DOpenLoopSample:
+    index: int
+    episode_index: int
+    task: str
+    state_raw: torch.Tensor
+    action_raw: torch.Tensor
+    inputs: dict[str, torch.Tensor]
+
+
+class A2DOpenLoopDataset:
+    image_keys = [
+        "observation.images.head",
+        "observation.images.hand_left",
+        "observation.images.hand_right",
+    ]
+    state_keys = [
+        "observation.states.joint.position",
+        "observation.states.effector.position",
+    ]
+    action_keys = [
+        "actions.joint.position",
+        "actions.effector.position",
+    ]
+
+    def __init__(
+        self,
+        dataset_root: str | Path,
+        *,
+        config: InternVLAA1Config,
+        train_stats: dict[str, Any],
+        image_offsets: tuple[int, int] = (-15, 0),
+        tolerance_s: float = 1e-4,
+    ) -> None:
+        self.root = Path(dataset_root)
+        self.config = config
+        self.info = _load_json(self.root / "meta" / "info.json")
+        self.dataset_stats = _load_json(self.root / "meta" / "stats.json")
+        self.data_rows = _load_parquet_rows(self.root / "data" / "chunk-000" / "file-000.parquet")
+        self.episode_rows = _load_parquet_rows(self.root / "meta" / "episodes" / "chunk-000" / "file-000.parquet")
+        self.task_rows = _load_parquet_rows(self.root / "meta" / "tasks.parquet")
+
+        self.state_stats = _stack_stats(train_stats, self.state_keys)
+        self.action_stats = _stack_stats(train_stats, self.action_keys)
+        self.image_offsets = image_offsets
+        self.tolerance_s = tolerance_s
+        self.video_reader = TorchvisionVideoReaderCache(backend="pyav")
+
+    @property
+    def num_episodes(self) -> int:
+        return len(self.episode_rows)
+
+    @property
+    def physical_action_dim(self) -> int:
+        return 16
+
+    def episode_start_indices(self, max_episodes: int | None = None) -> list[tuple[int, list[int]]]:
+        rows = self.episode_rows if max_episodes is None else self.episode_rows[:max_episodes]
+        result = []
+        for ep in rows:
+            start = int(ep["dataset_from_index"])
+            end = int(ep["dataset_to_index"])
+            result.append((int(ep["episode_index"]), list(range(start, end, self.config.chunk_size))))
+        return result
+
+    def _task_text(self, task_index: int) -> str:
+        return self.task_rows[task_index]["__index_level_0__"]
+
+    def _episode_for_index(self, row: dict[str, Any]) -> dict[str, Any]:
+        return self.episode_rows[int(row["episode_index"])]
+
+    def _state_vector(self, row: dict[str, Any]) -> torch.Tensor:
+        return torch.tensor(row[self.state_keys[0]] + row[self.state_keys[1]], dtype=torch.float32)
+
+    def _action_vector(self, row: dict[str, Any]) -> torch.Tensor:
+        return torch.tensor(row[self.action_keys[0]] + row[self.action_keys[1]], dtype=torch.float32)
+
+    def _query_rows(self, idx: int, deltas: list[int]) -> list[dict[str, Any]]:
+        row = self.data_rows[idx]
+        episode = self._episode_for_index(row)
+        start = int(episode["dataset_from_index"])
+        end = int(episode["dataset_to_index"])
+        return [self.data_rows[_clamp_index(idx + delta, start, end)] for delta in deltas]
+
+    def _decode_camera_history(
+        self, episode: dict[str, Any], camera_key: str, rows: list[dict[str, Any]]
+    ) -> torch.Tensor:
+        timestamps = [float(r["timestamp"]) for r in rows]
+        shifted = [float(episode[f"videos/{camera_key}/from_timestamp"]) + ts for ts in timestamps]
+        chunk_idx = int(episode[f"videos/{camera_key}/chunk_index"])
+        file_idx = int(episode[f"videos/{camera_key}/file_index"])
+        path = self.root / self.info["video_path"].format(
+            video_key=camera_key,
+            chunk_index=chunk_idx,
+            file_index=file_idx,
+        )
+        frames = self.video_reader.decode_frames(str(path), shifted, tolerance_s=self.tolerance_s)
+        return frames
+
+    def get_sample(self, idx: int) -> A2DOpenLoopSample:
+        row = self.data_rows[idx]
+        episode = self._episode_for_index(row)
+        image_rows = self._query_rows(idx, list((-15, 0)))
+        action_rows = self._query_rows(idx, list(range(self.config.chunk_size)))
+        camera_images = [self._decode_camera_history(episode, camera_key, image_rows) for camera_key in self.image_keys]
+        state_raw = self._state_vector(row)
+        state_norm = normalize_vector(state_raw, self.state_stats)
+        action_raw = torch.stack([self._action_vector(action_row) for action_row in action_rows], dim=0)
+        task = self._task_text(int(row["task_index"]))
+        inputs = {
+            OBS_STATE: state_norm,
+            OBS_TASK: task,
+            f"{OBS_IMAGES}.image0": camera_images[0],
+            f"{OBS_IMAGES}.image1": camera_images[1],
+            f"{OBS_IMAGES}.image2": camera_images[2],
+            f"{OBS_IMAGES}.image0_mask": torch.tensor(True),
+            f"{OBS_IMAGES}.image1_mask": torch.tensor(True),
+            f"{OBS_IMAGES}.image2_mask": torch.tensor(True),
+        }
+        return A2DOpenLoopSample(
+            index=idx,
+            episode_index=int(row["episode_index"]),
+            task=task,
+            state_raw=state_raw,
+            action_raw=action_raw,
+            inputs=inputs,
+        )
+
+
+def collate_open_loop_samples(
+    samples: list[A2DOpenLoopSample],
+    *,
+    device: str,
+    dtype: torch.dtype,
+) -> tuple[dict[str, torch.Tensor], dict[str, Any]]:
+    first = samples[0]
+    batch_inputs: dict[str, torch.Tensor] = {}
+    for key in first.inputs:
+        values = [sample.inputs[key] for sample in samples]
+        if isinstance(values[0], torch.Tensor):
+            tensor = torch.stack(values, dim=0)
+            if tensor.dtype in (torch.int64, torch.bool):
+                batch_inputs[key] = tensor.to(device=device)
+            else:
+                batch_inputs[key] = tensor.to(device=device, dtype=dtype)
+        else:
+            batch_inputs[key] = values
+
+    metadata = {
+        "indices": [sample.index for sample in samples],
+        "episode_indices": [sample.episode_index for sample in samples],
+        "tasks": [sample.task for sample in samples],
+        "state_raw": torch.stack([sample.state_raw for sample in samples], dim=0),
+        "action_raw": torch.stack([sample.action_raw for sample in samples], dim=0),
+    }
+    return batch_inputs, metadata
+
+
+def plot_prediction_series(
+    *,
+    series: dict[str, np.ndarray],
+    output_path: Path,
+    title: str,
+) -> None:
+    import matplotlib.pyplot as plt
+
+    first_series = next(iter(series.values()))
+    num_dims = int(first_series.shape[1])
+    num_cols = 2
+    num_rows = math.ceil(num_dims / num_cols)
+    fig, axs = plt.subplots(num_rows, num_cols, figsize=(16, max(6, 3 * num_rows)))
+    axs = np.array(axs).reshape(-1)
+    x_values = np.arange(first_series.shape[0])
+    styles = [
+        ("Ground Truth", "blue", "-", 1.5),
+        ("VLLM", "green", "-.", 1.4),
+        ("Predicted", "red", "--", 1.4),
+    ]
+    style_map = {name: (color, linestyle, linewidth) for name, color, linestyle, linewidth in styles}
+
+    for dim in range(num_dims):
+        ax = axs[dim]
+        for name, values in series.items():
+            color, linestyle, linewidth = style_map.get(name, (None, "-", 1.2))
+            ax.plot(
+                x_values,
+                values[:, dim],
+                label=name,
+                color=color,
+                linestyle=linestyle,
+                linewidth=linewidth,
+            )
+        ax.set_title(f"Dimension {dim + 1}")
+        ax.set_xlabel("Time Step / Sample Index")
+        ax.set_ylabel(f"Value Dim {dim + 1}")
+        ax.legend(loc="upper right")
+        ax.grid(True, linestyle="--", alpha=0.7)
+
+    for dim in range(num_dims, len(axs)):
+        axs[dim].axis("off")
+
+    fig.suptitle(title, fontsize=16)
+    plt.tight_layout(rect=(0, 0, 1, 0.97))
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    plt.savefig(output_path)
+    plt.close(fig)
+
+
+def summarize_prediction_metrics(
+    *,
+    gt_tensor: torch.Tensor,
+    pred_tensor: torch.Tensor,
+    joint_dims: int = 14,
+) -> dict[str, float]:
+    return {
+        "mse": float(F.mse_loss(gt_tensor, pred_tensor, reduction="mean").item()),
+        "mae": float(F.l1_loss(gt_tensor, pred_tensor, reduction="mean").item()),
+        "mse_joint": float(F.mse_loss(gt_tensor[:, :joint_dims], pred_tensor[:, :joint_dims], reduction="mean").item()),
+        "mae_joint": float(F.l1_loss(gt_tensor[:, :joint_dims], pred_tensor[:, :joint_dims], reduction="mean").item()),
+        "mse_gripper": float(
+            F.mse_loss(gt_tensor[:, joint_dims:], pred_tensor[:, joint_dims:], reduction="mean").item()
+        ),
+        "mae_gripper": float(
+            F.l1_loss(gt_tensor[:, joint_dims:], pred_tensor[:, joint_dims:], reduction="mean").item()
+        ),
+    }
+
+
+def summarize_metric_list(values: list[float]) -> float | None:
+    return float(np.mean(values)) if values else None
+
+
+def run_open_loop_evaluation(
+    *,
+    mode: str,
+    policy,
+    config,
+    dataset,
+    train_meta,
+    collate_samples: Callable[..., tuple[dict[str, torch.Tensor], dict[str, Any]]],
+    run_sample_actions: Callable[[Any, dict[str, torch.Tensor], torch.Tensor], torch.Tensor],
+    num_episodes: int,
+    seed: int,
+    device: str,
+    dtype: torch.dtype,
+    output_dir: str | Path,
+    skip_plots: bool,
+) -> dict[str, Any]:
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    plots_dir = output_dir / "plots"
+    plots_dir.mkdir(parents=True, exist_ok=True)
+
+    mse_values: list[float] = []
+    mae_values: list[float] = []
+    mse_joint_values: list[float] = []
+    mae_joint_values: list[float] = []
+    mse_gripper_values: list[float] = []
+    mae_gripper_values: list[float] = []
+    per_episode: list[dict[str, Any]] = []
+
+    plotting_available = True
+    if not skip_plots:
+        try:
+            import matplotlib.pyplot as _plt  # noqa: F401
+        except ModuleNotFoundError:
+            plotting_available = False
+            print("[warning] matplotlib is not installed; plots will be skipped.")
+
+    episode_specs = dataset.episode_start_indices(max_episodes=num_episodes)
+    for episode_index, indices in episode_specs:
+        print(f"[{mode}] episode: {episode_index}")
+        pred_chunks = []
+        gt_chunks = []
+        visited_indices = []
+        task = None
+
+        for index in indices:
+            sample = dataset.get_sample(index)
+            task = sample.task
+            batch_inputs, meta = collate_samples([sample], device=device, dtype=dtype)
+            noise = make_shared_noise(
+                seed,
+                index,
+                (batch_inputs[OBS_STATE].shape[0], config.chunk_size, config.max_action_dim),
+                device,
+            )
+            with torch.no_grad():
+                pred = run_sample_actions(policy, batch_inputs, noise)
+
+            pred = pred[:, :, : dataset.physical_action_dim].to(torch.float32).cpu()
+            pred_phys = unnormalize_vector(pred, dataset.action_stats)
+            if train_meta.action_mode == "delta":
+                pred_phys[:, :, :14] += meta["state_raw"][:, None, :14]
+
+            gt_phys = meta["action_raw"][:, :, : dataset.physical_action_dim].to(torch.float32).cpu()
+            pred_chunks.append(pred_phys[0])
+            gt_chunks.append(gt_phys[0])
+            visited_indices.append(index)
+
+        pred_tensor = torch.cat(pred_chunks, dim=0)
+        gt_tensor = torch.cat(gt_chunks, dim=0)
+        metrics = summarize_prediction_metrics(gt_tensor=gt_tensor, pred_tensor=pred_tensor)
+
+        mse_values.append(metrics["mse"])
+        mae_values.append(metrics["mae"])
+        mse_joint_values.append(metrics["mse_joint"])
+        mae_joint_values.append(metrics["mae_joint"])
+        mse_gripper_values.append(metrics["mse_gripper"])
+        mae_gripper_values.append(metrics["mae_gripper"])
+
+        if not skip_plots and plotting_available:
+            plot_prediction_series(
+                series={
+                    "Ground Truth": gt_tensor.numpy(),
+                    "VLLM": pred_tensor.numpy(),
+                },
+                output_path=plots_dir / f"{mode}_open_loop_ep{episode_index}.jpg",
+                title=f"{mode} Ground Truth vs VLLM",
+            )
+
+        episode_log = {
+            "episode_id": int(episode_index),
+            "task": task,
+            "visited_indices": visited_indices,
+            "num_pred_steps": int(pred_tensor.shape[0]),
+            **metrics,
+        }
+        per_episode.append(episode_log)
+        print(json.dumps({"mode": mode, **episode_log}, ensure_ascii=False))
+
+    summary = {
+        "mode": mode,
+        "num_episodes": len(per_episode),
+        "mse": mse_values,
+        "mae": mae_values,
+        "average_mse": summarize_metric_list(mse_values),
+        "average_mae": summarize_metric_list(mae_values),
+        "mse_joint": mse_joint_values,
+        "mae_joint": mae_joint_values,
+        "average_mse_joint": summarize_metric_list(mse_joint_values),
+        "average_mae_joint": summarize_metric_list(mae_joint_values),
+        "mse_gripper": mse_gripper_values,
+        "mae_gripper": mae_gripper_values,
+        "average_mse_gripper": summarize_metric_list(mse_gripper_values),
+        "average_mae_gripper": summarize_metric_list(mae_gripper_values),
+        "episodes": per_episode,
+    }
+    with open(output_dir / "log.json", "w", encoding="utf-8") as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+    return summary

examples/offline_inference/internvla_a1/run.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$ROOT_DIR/../../.." && pwd)"
+ : "${INTERNVLA_A1_MODEL_DIR:?Please export INTERNVLA_A1_MODEL_DIR=/path/to/InternVLA-A1-3B-ft-pen}"
+ : "${INTERNVLA_A1_DATASET_DIR:?Please export INTERNVLA_A1_DATASET_DIR=/path/to/Genie1-Place_Markpen}"
+ : "${INTERNVLA_A1_PROCESSOR_DIR:?Please export INTERNVLA_A1_PROCESSOR_DIR=/path/to/Qwen3-VL-2B-Instruct}"
+ : "${INTERNVLA_A1_COSMOS_DIR:?Please export INTERNVLA_A1_COSMOS_DIR=/path/to/Cosmos-Tokenizer-CI8x8-SafeTensor}"
+INTERNVLA_A1_OUTPUT_DIR="${INTERNVLA_A1_OUTPUT_DIR:-$REPO_ROOT/outputs/internvla_a1/vllm_infer}"
+
+export INTERNVLA_A1_MODEL_DIR
+export INTERNVLA_A1_DATASET_DIR
+export INTERNVLA_A1_PROCESSOR_DIR
+export INTERNVLA_A1_COSMOS_DIR
+
+python "$ROOT_DIR/end2end.py" \
+  --model-dir "$INTERNVLA_A1_MODEL_DIR" \
+  --dataset-dir "$INTERNVLA_A1_DATASET_DIR" \
+  --output-dir "$INTERNVLA_A1_OUTPUT_DIR" \
+  --num-episodes "${INTERNVLA_A1_NUM_EPISODES:-1}" \
+  --attn-implementation eager \
+  "$@"

examples/offline_inference/moss_tts_nano/README.md

@@ -0,0 +1,101 @@
+# MOSS-TTS-Nano Offline Inference
+
+## Overview
+
+Single-stage offline TTS pipeline using the 0.1B MOSS-TTS-Nano AR LM and MOSS-Audio-Tokenizer-Nano codec. Outputs 48 kHz stereo WAV.
+
+## Quick Start
+
+```bash
+python end2end.py --text "Hello, this is MOSS-TTS-Nano."
+```
+
+The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
+
+## Usage
+
+```
+python end2end.py [OPTIONS]
+
+Options:
+  --text TEXT               Text to synthesize (default: "Hello, this is MOSS-TTS-Nano speaking.")
+  --voice VOICE             Built-in voice preset (default: Junhao). See voice table below.
+  --mode MODE               voice_clone (default) or continuation
+  --prompt-audio PATH       Reference WAV/MP3 for custom voice cloning
+  --prompt-text TEXT        Reference transcript (continuation mode)
+  --max-new-frames N        Max AR frames, default 375 (~14 s audio)
+  --seed INT                Random seed for reproducibility
+  --audio-temperature F     Audio sampling temperature (default: 0.8)
+  --audio-top-k N           Audio top-k sampling (default: 25)
+  --audio-top-p F           Audio top-p sampling (default: 0.95)
+  --text-temperature F      Text layer temperature (default: 1.0)
+  --batch                   Run a built-in batch of diverse samples (ZH/EN/FR)
+  --output-dir DIR          Directory for WAV outputs (default: /tmp/moss_tts_nano_output)
+  --deploy-config PATH      Override deploy YAML (defaults to vllm_omni/deploy/moss_tts_nano.yaml)
+  --stage-init-timeout INT  Timeout in seconds for stage init (default: 120)
+```
+
+## Examples
+
+```bash
+# Built-in Chinese voice
+python end2end.py --text "你好，这是MOSS-TTS-Nano的语音合成演示。" --voice Junhao
+
+# Built-in English voice
+python end2end.py --text "Hello from MOSS-TTS-Nano." --voice Ava
+
+# Custom voice clone
+python end2end.py \
+    --text "Hello, this is a cloned voice." \
+    --prompt-audio /path/to/reference.wav \
+    --prompt-text "Exact transcript of the reference audio."
+
+# Batch synthesis (ZH/EN/FR)
+python end2end.py --batch --output-dir /tmp/batch_output
+
+# Reproducible output
+python end2end.py --text "Deterministic test." --seed 42
+```
+
+## Built-in Voice Presets
+
+| Voice | Language |
+|-------|----------|
+| `Junhao` | ZH |
+| `Zhiming` | ZH |
+| `Weiguo` | ZH |
+| `Xiaoyu` | ZH |
+| `Yuewen` | ZH |
+| `Lingyu` | ZH |
+| `Ava` | EN |
+| `Bella` | EN |
+| `Adam` | EN |
+| `Nathan` | EN |
+| `Sakura` | JA |
+| `Yui` | JA |
+| `Aoi` | JA |
+| `Hina` | JA |
+| `Mei` | JA |
+
+## Deploy Config
+
+Runtime knobs live in `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded;
+override with `--deploy-config PATH`). Key stage-level settings:
+
+```yaml
+stages:
+  - stage_id: 0
+    gpu_memory_utilization: 0.3   # ~2 GB VRAM; increase for faster init
+    max_num_seqs: 4               # concurrent requests
+    max_model_len: 4096
+```
+
+## Output Format
+
+WAV files, 48 kHz, stereo (2-channel). The codec interleaves stereo as `[L, R, L, R, ...]` in the flat tensor returned by the model.
+
+## Troubleshooting
+
+- **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ torchcodec backend requires NVRTC. The model patches `torchaudio.load/save` automatically at load time to fall back to soundfile.
+- **`flash_attn not installed`**: The model falls back to `sdpa` attention automatically.
+- **Empty audio**: Check that `--text` is non-empty and the model loaded successfully (look for "MOSS-TTS-Nano LM loaded" in logs).

examples/offline_inference/moss_tts_nano/end2end.py

@@ -0,0 +1,180 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline inference example for MOSS-TTS-Nano via vLLM-Omni.
+
+Single-stage pipeline: the 0.1B AR LM and MOSS-Audio-Tokenizer-Nano codec
+both run inside one generation stage.  Output is 48 kHz stereo WAV.
+
+Supports:
+  - Built-in voice presets (--voice)
+  - Custom voice cloning via reference audio (--prompt-audio)
+  - Batch synthesis of multiple texts
+  - Reproducible output via --seed
+
+Usage examples:
+  # Built-in voice
+  python end2end.py --text "Hello from MOSS-TTS-Nano."
+
+  # Voice clone with reference audio
+  python end2end.py --text "Hello!" --prompt-audio /path/to/ref.wav
+
+  # Batch with different voices
+  python end2end.py --batch --output-dir /tmp/moss_output
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import soundfile as sf
+import torch
+from vllm import SamplingParams
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+# Prevent multiprocessing from re-importing CUDA in the wrong context.
+os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+from vllm_omni import Omni  # noqa: E402
+
+MODEL = "OpenMOSS-Team/MOSS-TTS-Nano"
+
+BATCH_SAMPLES = [
+    {"text": "Hello, this is a test of MOSS-TTS-Nano.", "voice": "Ava", "label": "en_hello"},
+    {"text": "你好，这是 MOSS-TTS-Nano 的语音合成测试。", "voice": "Junhao", "label": "zh_hello"},
+    {"text": "Bonjour, ceci est un test de synthèse vocale.", "voice": "Bella", "label": "fr_hello"},
+]
+
+
+def build_request(
+    text: str,
+    voice: str = "Junhao",
+    mode: str = "voice_clone",
+    prompt_audio_path: str | None = None,
+    prompt_text: str | None = None,
+    max_new_frames: int = 375,
+    seed: int | None = None,
+    audio_temperature: float = 0.8,
+    audio_top_k: int = 25,
+    audio_top_p: float = 0.95,
+    text_temperature: float = 1.0,
+) -> dict:
+    """Build an Omni request payload for MOSS-TTS-Nano."""
+    additional: dict = {
+        "text": [text],
+        "voice": [voice],
+        "mode": [mode],
+        "max_new_frames": [max_new_frames],
+        "audio_temperature": [audio_temperature],
+        "audio_top_k": [audio_top_k],
+        "audio_top_p": [audio_top_p],
+        "text_temperature": [text_temperature],
+    }
+    if prompt_audio_path:
+        additional["prompt_audio_path"] = [str(prompt_audio_path)]
+    if prompt_text:
+        additional["prompt_text"] = [prompt_text]
+    if seed is not None:
+        additional["seed"] = [seed]
+
+    return {
+        "prompt": "<|im_start|>assistant\n",  # minimal placeholder prompt
+        "additional_information": additional,
+    }
+
+
+def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 48000) -> None:
+    audio_np = waveform.float().numpy()
+    # Reshape stereo: inference_stream yields interleaved [samples*2]; reshape to [samples, 2]
+    if audio_np.ndim == 1 and audio_np.shape[0] % 2 == 0:
+        audio_np = audio_np.reshape(-1, 2)
+    sf.write(path, audio_np, sample_rate)
+    print(f"  Saved {path} ({audio_np.shape}, {sample_rate} Hz)")
+
+
+def main(args) -> None:
+    omni = Omni(
+        model=MODEL,
+        deploy_config=args.deploy_config,
+        stage_init_timeout=args.stage_init_timeout,
+    )
+
+    sampling_params = SamplingParams(
+        temperature=1.0,
+        top_p=1.0,
+        top_k=50,
+        max_tokens=4096,
+        seed=args.seed if args.seed is not None else 42,
+        detokenize=False,
+    )
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if args.batch:
+        print(f"Running batch synthesis ({len(BATCH_SAMPLES)} samples)...")
+        inputs = [build_request(s["text"], voice=s["voice"], seed=args.seed) for s in BATCH_SAMPLES]
+        params_list = [sampling_params] * len(inputs)
+    else:
+        print(f"Synthesizing: {args.text!r}")
+        inputs = build_request(
+            text=args.text,
+            voice=args.voice,
+            mode=args.mode,
+            prompt_audio_path=args.prompt_audio,
+            prompt_text=args.prompt_text,
+            max_new_frames=args.max_new_frames,
+            seed=args.seed,
+            audio_temperature=args.audio_temperature,
+            audio_top_k=args.audio_top_k,
+            audio_top_p=args.audio_top_p,
+            text_temperature=args.text_temperature,
+        )
+        params_list = sampling_params
+
+    for stage_outputs in omni.generate(inputs, params_list):
+        for i, req_output in enumerate(stage_outputs.request_output):
+            for j, out in enumerate(req_output.outputs):
+                mm = out.multimodal_output
+                if mm is None:
+                    print(f"  [req {i}] No audio output.")
+                    continue
+                audio = mm.get("audio")
+                sr_tensor = mm.get("sr")
+                if audio is None:
+                    print(f"  [req {i}] No waveform in multimodal_output.")
+                    continue
+                sr = int(sr_tensor.item()) if sr_tensor is not None else 48000
+                label = BATCH_SAMPLES[i]["label"] if args.batch else f"output_{i}_{j}"
+                out_path = str(output_dir / f"{label}.wav")
+                save_audio(audio.cpu(), out_path, sr)
+
+    print("Done.")
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="MOSS-TTS-Nano offline inference")
+    parser.add_argument("--text", default="Hello, this is MOSS-TTS-Nano speaking.", help="Text to synthesize.")
+    parser.add_argument("--voice", default="Junhao", help="Built-in voice preset name.")
+    parser.add_argument("--mode", default="voice_clone", choices=["voice_clone", "continuation"])
+    parser.add_argument("--prompt-audio", default=None, help="Path to reference audio for voice cloning.")
+    parser.add_argument("--prompt-text", default=None, help="Reference transcript (continuation mode).")
+    parser.add_argument("--max-new-frames", type=int, default=375, help="Max AR frames (~14s at default).")
+    parser.add_argument("--seed", type=int, default=None, help="Random seed.")
+    parser.add_argument("--audio-temperature", type=float, default=0.8)
+    parser.add_argument("--audio-top-k", type=int, default=25)
+    parser.add_argument("--audio-top-p", type=float, default=0.95)
+    parser.add_argument("--text-temperature", type=float, default=1.0)
+    parser.add_argument("--batch", action="store_true", help="Run built-in batch of diverse samples.")
+    parser.add_argument("--output-dir", default="/tmp/moss_tts_nano_output", help="Directory for WAV outputs.")
+    parser.add_argument(
+        "--deploy-config",
+        default=None,
+        help="Path to a deploy YAML; leave unset to auto-load vllm_omni/deploy/moss_tts_nano.yaml.",
+    )
+    parser.add_argument("--stage-init-timeout", type=int, default=120)
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main(parse_args())

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -242,8 +242,7 @@ async def run_single_request(
                 if stage_0_first_output_ts is None:
                     stage_0_first_output_ts = time.perf_counter()
                 text_output = output.outputs[0].text
-                if output.finished:
-                    text_parts.append(text_output)
+                text_parts.append(text_output)
             elif omni_output.final_output_type == "audio":
                 mm_out = output.outputs[0].multimodal_output
                 if mm_out and "audio" in mm_out:
@@ -293,7 +292,7 @@ async def run_single_request(
     if text_parts:
         text_file = os.path.join(output_dir, f"{request_id}.txt")
         with open(text_file, "w", encoding="utf-8") as f:
-            f.write("\n".join(text_parts))
+            f.write("".join(text_parts))
         result["saved_files"].append(text_file)
         print(
             f"[Request {request_id}] Text saved to {text_file} "
@@ -393,6 +392,10 @@ async def run_all(args):
 
         # Use default sampling params from stage config (they are pre-configured
         # in the YAML for each stage).
+        #
+        # NOTE: Since we do not set the sampling params directly, .generate in
+        # will automatically set the output kind to delta, since this is what
+        # makes sense for most multimodal use-cases.
         sampling_params_list = None
 
         output_dir = args.output_dir

examples/offline_inference/text_to_image/text_to_image.py

@@ -300,6 +300,7 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help=("Custom system prompt. Used when --use-system-prompt is custom. "),
     )
+    current_omni_platform.pre_register_and_update(parser)
     from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
     nullify_stage_engine_defaults(parser)

examples/online_serving/moss_tts_nano/README.md

@@ -0,0 +1,158 @@
+# MOSS-TTS-Nano
+
+## Model checkpoint
+
+| Model | Description |
+|-------|-------------|
+| `OpenMOSS-Team/MOSS-TTS-Nano` | 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec, 48 kHz stereo, ZH/EN/JA |
+
+## Gradio Demo
+
+An interactive Gradio demo is available with multilingual voice presets, custom voice cloning, and streaming support.
+
+```bash
+# Option 1: Launch server + Gradio together
+./run_gradio_demo.sh
+
+# Option 2: If server is already running
+python gradio_demo.py --api-base http://localhost:8091
+```
+
+Then open http://localhost:7860 in your browser.
+
+Features:
+
+- 15 built-in voice presets (6 ZH · 4 EN · 5 JA)
+- Custom voice cloning from uploaded reference audio
+- Streaming mode (progressive PCM playback)
+
+## Launch the Server
+
+```bash
+vllm serve OpenMOSS-Team/MOSS-TTS-Nano --omni --port 8091
+```
+
+The deploy config at `vllm_omni/deploy/moss_tts_nano.yaml` auto-loads; no
+`--stage-configs-path`, `--trust-remote-code`, or `--enforce-eager` flags
+are needed.
+
+Or use the convenience script:
+
+```bash
+./run_server.sh
+```
+
+## Send TTS Request
+
+### Using curl
+
+```bash
+# Built-in voice preset
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "你好，这是语音合成测试。",
+        "voice": "Junhao",
+        "response_format": "wav"
+    }' --output output.wav
+
+# English preset
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is MOSS-TTS-Nano.",
+        "voice": "Ava",
+        "response_format": "wav"
+    }' --output output_en.wav
+```
+
+### Custom Voice Cloning
+
+Provide a reference audio (base64 data URL) and its transcript:
+
+```bash
+REF_AUDIO=$(base64 -w 0 /path/to/reference.wav)
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"input\": \"Hello, this is a cloned voice.\",
+        \"voice\": \"Junhao\",
+        \"ref_audio\": \"data:audio/wav;base64,${REF_AUDIO}\",
+        \"ref_text\": \"Exact transcript of the reference audio.\"
+    }" --output cloned.wav
+```
+
+### Using Python
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/speech",
+    json={
+        "input": "你好，这是语音合成测试。",
+        "voice": "Junhao",
+        "response_format": "wav",
+    },
+    timeout=300.0,
+)
+
+with open("output.wav", "wb") as f:
+    f.write(response.content)
+```
+
+### Streaming
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, streaming output from MOSS-TTS-Nano.",
+        "voice": "Ava",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 2 -
+```
+
+**Note:** MOSS-TTS-Nano outputs at 48 kHz stereo (2-channel).
+
+## API Parameters
+
+MOSS-TTS-Nano uses the standard `/v1/audio/speech` endpoint.
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input` | string | **required** | Text to synthesize (ZH / EN / JA) |
+| `voice` | string | `"Junhao"` | Built-in voice name (see table below) |
+| `response_format` | string | `"wav"` | Audio format: wav, mp3, flac, pcm |
+| `ref_audio` | string | null | Base64 data URL for custom voice cloning |
+| `ref_text` | string | null | Transcript of reference audio (required when `ref_audio` is set) |
+| `stream` | bool | false | Stream raw PCM chunks |
+| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
+
+## Built-in Voice Presets
+
+| Voice | Language | Description |
+|-------|----------|-------------|
+| `Junhao` | ZH | Male, standard Mandarin |
+| `Zhiming` | ZH | Male |
+| `Weiguo` | ZH | Male |
+| `Xiaoyu` | ZH | Female |
+| `Yuewen` | ZH | Female |
+| `Lingyu` | ZH | Female |
+| `Ava` | EN | Female, American English |
+| `Bella` | EN | Female |
+| `Adam` | EN | Male |
+| `Nathan` | EN | Male |
+| `Sakura` | JA | Female |
+| `Yui` | JA | Female |
+| `Aoi` | JA | Female |
+| `Hina` | JA | Female |
+| `Mei` | JA | Female |
+
+## Troubleshooting
+
+1. **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ defaults to torchcodec which requires NVRTC. vLLM-Omni patches this automatically at model load time to use soundfile instead.
+2. **Connection refused**: Ensure the server is running on the correct port.
+3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings.
+4. **Out of memory**: The default `gpu_memory_utilization=0.3` is conservative. Increase it in the stage config if you have more VRAM available.

examples/online_serving/moss_tts_nano/gradio_demo.py

@@ -0,0 +1,690 @@
+"""Gradio demo for MOSS-TTS-Nano with gapless streaming audio playback.
+
+Uses a custom AudioWorklet-based player (adapted from the Qwen3-TTS demo)
+for gap-free streaming.  Audio is streamed from the vLLM-Omni server
+through a same-origin proxy and played via the Web Audio API's AudioWorklet.
+
+MOSS-TTS-Nano outputs 48 kHz stereo (2-channel interleaved L,R,L,R,...).
+The AudioWorklet downmixes stereo to mono during playback.
+
+Also supports non-streaming mode (full audio download) via gr.Audio.
+
+Usage:
+    # Start the server first (see run_server.sh), then:
+    python gradio_demo.py --api-base http://localhost:8091
+
+    # Or use run_gradio_demo.sh to start both server and demo together.
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import json
+import logging
+
+try:
+    import gradio as gr
+except ImportError:
+    raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
+
+import httpx
+import numpy as np
+import soundfile as sf
+from fastapi import FastAPI, Request
+from fastapi.responses import Response, StreamingResponse
+
+logger = logging.getLogger(__name__)
+
+# MOSS-TTS-Nano outputs 48 kHz stereo (2-channel interleaved).
+PCM_SAMPLE_RATE = 48000
+PCM_CHANNELS = 2
+DEFAULT_API_BASE = "http://localhost:8091"
+
+# Built-in voice presets (matches the MOSS-TTS-Nano GitHub repo assets/)
+VOICE_PRESETS = {
+    # Chinese
+    "Junhao (ZH)": "Junhao",
+    "Zhiming (ZH)": "Zhiming",
+    "Weiguo (ZH)": "Weiguo",
+    "Xiaoyu (ZH)": "Xiaoyu",
+    "Yuewen (ZH)": "Yuewen",
+    "Lingyu (ZH)": "Lingyu",
+    # English
+    "Ava (EN)": "Ava",
+    "Bella (EN)": "Bella",
+    "Adam (EN)": "Adam",
+    "Nathan (EN)": "Nathan",
+    # Japanese
+    "Sakura (JA)": "Sakura",
+    "Yui (JA)": "Yui",
+    "Aoi (JA)": "Aoi",
+    "Hina (JA)": "Hina",
+    "Mei (JA)": "Mei",
+}
+PRESET_LABELS = list(VOICE_PRESETS.keys())
+DEFAULT_PRESET = "Junhao (ZH)"
+
+# ── AudioWorklet processor (loaded in browser via Blob URL) ──────────
+# Downmixes stereo-interleaved int16 PCM to mono float32 for playback.
+WORKLET_JS = r"""
+class TTSPlaybackProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+        this.queue = [];
+        this.buf = null;
+        this.pos = 0;
+        this.playing = false;
+        this.played = 0;
+        this.port.onmessage = (e) => {
+            if (e.data && e.data.type === 'clear') {
+                this.queue = []; this.buf = null; this.pos = 0; this.played = 0;
+                if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped'}); }
+                return;
+            }
+            // Input: Int16Array of stereo-interleaved samples [L0,R0,L1,R1,...]
+            // Convert to mono float32 by averaging L+R channels.
+            const pcm = e.data;
+            const monoLen = Math.floor(pcm.length / 2);
+            const mono = new Float32Array(monoLen);
+            for (let i = 0; i < monoLen; i++) {
+                mono[i] = (pcm[i * 2] + pcm[i * 2 + 1]) / 65536.0;
+            }
+            this.queue.push(mono);
+        };
+    }
+    process(inputs, outputs) {
+        const out = outputs[0][0];
+        for (let i = 0; i < out.length; i++) {
+            if (!this.buf || this.pos >= this.buf.length) {
+                if (this.queue.length > 0) {
+                    this.buf = this.queue.shift(); this.pos = 0;
+                } else {
+                    for (let j = i; j < out.length; j++) out[j] = 0;
+                    if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped', played:this.played}); }
+                    return true;
+                }
+            }
+            out[i] = this.buf[this.pos++];
+            this.played++;
+        }
+        if (!this.playing) { this.playing = true; this.port.postMessage({type:'started'}); }
+        return true;
+    }
+}
+registerProcessor('tts-playback-processor', TTSPlaybackProcessor);
+"""
+
+# ── Player HTML (container with metric cards) ────────────────────────
+PLAYER_HTML = """
+<div id="tts-player">
+  <div style="display:flex; align-items:center; gap:10px;">
+    <div id="tts-status-dot" style="width:10px;height:10px;border-radius:50%;background:#ccc;flex-shrink:0;"></div>
+    <span id="tts-status" style="font-weight:600;font-size:1.05em;">Ready</span>
+    <button id="tts-stop-btn" onclick="window.ttsStop()"
+      style="display:none; margin-left:auto; padding:5px 16px; border-radius:6px; border:1px solid #EF5552;
+             background:#fff; color:#EF5552; cursor:pointer; font-size:0.85em; transition:all 0.15s;">Stop</button>
+  </div>
+  <div id="tts-metrics" style="display:none; grid-template-columns:repeat(4,1fr); gap:10px; margin-top:12px;">
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">TTFP</div>
+      <div id="tts-m-ttfp" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">RTF</div>
+      <div id="tts-m-rtf" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Audio</div>
+      <div id="tts-m-dur" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+    <div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
+      <div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Speed</div>
+      <div id="tts-m-speed" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
+    </div>
+  </div>
+  <div id="tts-rtf-bar-wrap" style="display:none; background:#e8ecf1; border-radius:4px; height:20px; overflow:hidden; position:relative; margin-top:10px;">
+    <div id="tts-rtf-bar" style="height:100%; border-radius:4px; transition:width 0.3s ease, background 0.3s ease; width:0%;"></div>
+    <span id="tts-rtf-label" style="position:absolute;top:50%;left:50%;transform:translate(-50%,-50%);font-size:0.75em;font-weight:600;color:#444;"></span>
+  </div>
+  <div id="tts-elapsed" style="display:none; margin-top:6px; font-size:0.8em; color:#999; text-align:right;"></div>
+</div>
+"""
+
+
+def _build_player_js(sample_rate: int) -> str:
+    """Build the JavaScript that powers the AudioWorklet player."""
+    return f"""
+    <script>
+    const SR = {sample_rate};
+    const WC = {json.dumps(WORKLET_JS)};
+    let ctx = null, node = null, abort = null, gen = false, st = {{}};
+
+    async function init() {{
+        if (ctx) return;
+        ctx = new AudioContext({{ sampleRate: SR }});
+        const b = new Blob([WC], {{ type: 'application/javascript' }});
+        const u = URL.createObjectURL(b);
+        await ctx.audioWorklet.addModule(u);
+        URL.revokeObjectURL(u);
+        node = new AudioWorkletNode(ctx, 'tts-playback-processor');
+        node.connect(ctx.destination);
+        node.port.onmessage = (e) => {{
+            if (e.data.type === 'started') setStatus('Playing...', '#2E7D32');
+            else if (e.data.type === 'stopped' && !gen) {{
+                setStatus('Done', '#2E7D32'); showStats(true);
+                const btn = document.getElementById('tts-stop-btn');
+                if (btn) btn.style.display = 'none';
+            }}
+        }};
+    }}
+
+    function setStatus(text, color) {{
+        const s = document.getElementById('tts-status');
+        const d = document.getElementById('tts-status-dot');
+        if (s) s.textContent = text;
+        if (d) d.style.background = color || '#ccc';
+    }}
+
+    function showStats(fin) {{
+        if (!st.t0) return;
+        const elapsed = (fin && st.streamEnd ? (st.streamEnd - st.t0) : (performance.now() - st.t0)) / 1000;
+        // st.samples counts stereo int16 samples; mono frames = samples / 2
+        const dur = (st.samples / 2) / SR;
+        const mTtfp = document.getElementById('tts-m-ttfp');
+        const mRtf = document.getElementById('tts-m-rtf');
+        const mDur = document.getElementById('tts-m-dur');
+        const mSpeed = document.getElementById('tts-m-speed');
+        const bar = document.getElementById('tts-rtf-bar');
+        const barLabel = document.getElementById('tts-rtf-label');
+        const elapsedEl = document.getElementById('tts-elapsed');
+
+        if (mTtfp && st.ttfp != null) mTtfp.textContent = st.ttfp.toFixed(0) + 'ms';
+        if (mDur) mDur.textContent = dur.toFixed(1) + 's';
+
+        if (dur > 0 && elapsed > 0) {{
+            const rtf = elapsed / dur;
+            const speed = 1 / rtf;
+            if (mRtf) {{
+                mRtf.textContent = rtf.toFixed(2) + 'x';
+                mRtf.style.color = rtf < 1 ? '#2E7D32' : rtf < 1.5 ? '#e8a317' : '#EF5552';
+            }}
+            if (mSpeed) {{
+                mSpeed.textContent = speed.toFixed(1) + 'x';
+                mSpeed.style.color = speed > 1 ? '#2E7D32' : speed > 0.7 ? '#e8a317' : '#EF5552';
+            }}
+            if (bar) {{
+                const pct = Math.min(speed / 3 * 100, 100);
+                bar.style.width = pct + '%';
+                bar.style.background = speed > 1 ? 'linear-gradient(90deg,#66BB6A,#2E7D32)' : speed > 0.7 ? 'linear-gradient(90deg,#e8a317,#f0b866)' : 'linear-gradient(90deg,#EF5552,#f87171)';
+            }}
+            if (barLabel) barLabel.textContent = speed.toFixed(1) + 'x realtime';
+        }}
+        if (elapsedEl) {{
+            elapsedEl.style.display = 'block';
+            elapsedEl.textContent = fin ? 'Completed in ' + elapsed.toFixed(1) + 's  (' + st.chunks + ' chunks)' : elapsed.toFixed(1) + 's elapsed  (' + st.chunks + ' chunks)';
+        }}
+    }}
+
+    window.ttsStop = function() {{
+        if (abort) abort.abort();
+        if (node) node.port.postMessage({{ type: 'clear' }});
+        gen = false;
+        setStatus('Stopped', '#999');
+        const btn = document.getElementById('tts-stop-btn');
+        if (btn) btn.style.display = 'none';
+    }};
+
+    window.ttsGenerate = async function(payload) {{
+        try {{ await init(); if (ctx.state === 'suspended') await ctx.resume(); }}
+        catch (e) {{ const s = document.getElementById('tts-status'); if (s) s.textContent = 'Audio init error: ' + e.message; return; }}
+
+        if (abort) abort.abort();
+        node.port.postMessage({{ type: 'clear' }});
+        await new Promise(r => setTimeout(r, 50));
+        node.port.postMessage({{ type: 'clear' }});
+
+        gen = true;
+        st = {{ t0: null, chunks: 0, samples: 0, ttfp: null }};
+        setStatus('Connecting...', '#2E7D32');
+        const bEl = document.getElementById('tts-stop-btn');
+        if (bEl) bEl.style.display = 'inline-block';
+        const mp = document.getElementById('tts-metrics');
+        if (mp) {{ mp.style.display = 'grid'; ['tts-m-ttfp','tts-m-rtf','tts-m-dur','tts-m-speed'].forEach(id => {{ const e = document.getElementById(id); if(e) {{ e.textContent = '\\u2014'; e.style.color = '#333'; }} }}); }}
+        const bw = document.getElementById('tts-rtf-bar-wrap');
+        if (bw) bw.style.display = 'block';
+        const bar = document.getElementById('tts-rtf-bar');
+        if (bar) bar.style.width = '0%';
+        const bl = document.getElementById('tts-rtf-label');
+        if (bl) bl.textContent = '';
+        const ee = document.getElementById('tts-elapsed');
+        if (ee) {{ ee.style.display = 'none'; ee.textContent = ''; }}
+        abort = new AbortController();
+
+        try {{
+            st.t0 = performance.now();
+            const r = await fetch('/proxy/v1/audio/speech', {{
+                method: 'POST',
+                headers: {{ 'Content-Type': 'application/json' }},
+                body: JSON.stringify(payload),
+                signal: abort.signal,
+            }});
+            if (!r.ok) {{ const t = await r.text(); throw new Error('Server ' + r.status + ': ' + t.slice(0, 200)); }}
+            setStatus('Streaming...', '#43A047');
+
+            const reader = r.body.getReader();
+            let left = new Uint8Array(0);
+            while (true) {{
+                const {{ done, value }} = await reader.read();
+                if (done) break;
+                let raw;
+                if (left.length > 0) {{
+                    raw = new Uint8Array(left.length + value.length);
+                    raw.set(left); raw.set(value, left.length);
+                }} else {{ raw = value; }}
+                const usable = raw.length - (raw.length % 2);
+                left = usable < raw.length ? raw.slice(usable) : new Uint8Array(0);
+                if (usable > 0) {{
+                    const ab = new ArrayBuffer(usable);
+                    new Uint8Array(ab).set(raw.subarray(0, usable));
+                    const pcm = new Int16Array(ab);
+                    node.port.postMessage(pcm);
+                    st.chunks++;
+                    st.samples += pcm.length;
+                    if (st.ttfp == null) st.ttfp = performance.now() - st.t0;
+                    showStats(false);
+                }}
+            }}
+        }} catch (e) {{
+            if (e.name !== 'AbortError') {{
+                setStatus('Error: ' + e.message, '#EF5552');
+                console.error('TTS error:', e);
+            }}
+        }} finally {{
+            st.streamEnd = performance.now();
+            showStats(true);
+            gen = false;
+            if (st.samples > 0) setStatus('Finishing playback...', '#2E7D32');
+            else {{
+                setStatus('No audio received', '#999');
+                if (bEl) bEl.style.display = 'none';
+            }}
+        }}
+    }};
+    </script>
+"""
+
+
+def encode_audio_to_base64(audio_data: tuple) -> str:
+    """Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
+    sample_rate, audio_np = audio_data
+    if audio_np.dtype != np.int16:
+        if audio_np.dtype in (np.float32, np.float64):
+            audio_np = np.clip(audio_np, -1.0, 1.0)
+            audio_np = (audio_np * 32767).astype(np.int16)
+        else:
+            audio_np = audio_np.astype(np.int16)
+    buf = io.BytesIO()
+    sf.write(buf, audio_np, sample_rate, format="WAV")
+    wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+    return f"data:audio/wav;base64,{wav_b64}"
+
+
+def build_payload(
+    text: str,
+    voice_preset: str,
+    ref_audio: tuple | None,
+    ref_text: str,
+    response_format: str = "pcm",
+    stream: bool = True,
+) -> dict:
+    """Build the /v1/audio/speech request payload."""
+    if not text or not text.strip():
+        raise gr.Error("Please enter text to synthesize.")
+
+    voice_name = VOICE_PRESETS.get(voice_preset, "Junhao")
+
+    payload: dict = {
+        "input": text.strip(),
+        "voice": voice_name,
+        "response_format": "pcm" if stream else response_format,
+        "stream": stream,
+    }
+
+    has_ref = ref_audio is not None
+    if has_ref:
+        payload["ref_audio"] = encode_audio_to_base64(ref_audio)
+        ref_text_stripped = ref_text.strip() if ref_text else ""
+        if ref_text_stripped:
+            payload["ref_text"] = ref_text_stripped
+        else:
+            raise gr.Error("Voice cloning requires a transcript of the reference audio.")
+
+    return payload
+
+
+def generate_speech(
+    api_base: str,
+    text: str,
+    voice_preset: str,
+    ref_audio: tuple | None,
+    ref_text: str,
+    response_format: str,
+) -> tuple:
+    """Non-streaming: call /v1/audio/speech and return full audio as (sr, np_array)."""
+    payload = build_payload(text, voice_preset, ref_audio, ref_text, response_format, stream=False)
+
+    try:
+        with httpx.Client(timeout=300.0) as client:
+            resp = client.post(
+                f"{api_base}/v1/audio/speech",
+                json=payload,
+                headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
+            )
+    except httpx.TimeoutException:
+        raise gr.Error("Request timed out. The server may be busy.")
+    except httpx.ConnectError:
+        raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
+
+    if resp.status_code != 200:
+        raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
+
+    try:
+        if response_format == "pcm":
+            # PCM is stereo interleaved; downmix to mono for Gradio.
+            samples = np.frombuffer(resp.content, dtype=np.int16)
+            n = len(samples) - (len(samples) % 2)
+            left = samples[:n:2].astype(np.float32)
+            right = samples[1:n:2].astype(np.float32)
+            mono = (left + right) / 65536.0
+            return (PCM_SAMPLE_RATE, mono)
+        # WAV/MP3/FLAC: soundfile handles header parsing.
+        audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
+        if audio_np.ndim > 1:
+            audio_np = audio_np.mean(axis=1)
+        return (int(sample_rate), audio_np.astype(np.float32))
+    except Exception as e:
+        raise gr.Error(f"Failed to decode audio: {e}")
+
+
+def create_app(api_base: str):
+    """Create the FastAPI app with streaming proxy + Gradio UI."""
+    fastapi_app = FastAPI()
+
+    # ── Streaming proxy (same-origin, no CORS issues) ────────────
+    @fastapi_app.post("/proxy/v1/audio/speech")
+    async def proxy_speech(request: Request):
+        body = await request.json()
+        logger.info(
+            "Proxy request: input=%r voice=%s stream=%s",
+            body.get("input", "")[:50],
+            body.get("voice"),
+            body.get("stream"),
+        )
+        try:
+            client = httpx.AsyncClient(timeout=300)
+            resp = await client.send(
+                client.build_request(
+                    "POST",
+                    f"{api_base}/v1/audio/speech",
+                    json=body,
+                    headers={"Authorization": "Bearer EMPTY", "Content-Type": "application/json"},
+                ),
+                stream=True,
+            )
+        except Exception as exc:
+            logger.exception("Proxy connection error")
+            await client.aclose()
+            return Response(content=str(exc), status_code=502)
+
+        if resp.status_code != 200:
+            content = await resp.aread()
+            logger.error("Proxy upstream error %d: %s", resp.status_code, content[:200])
+            await resp.aclose()
+            await client.aclose()
+            return Response(content=content, status_code=resp.status_code)
+
+        async def relay():
+            total = 0
+            try:
+                async for chunk in resp.aiter_bytes():
+                    total += len(chunk)
+                    yield chunk
+            except Exception:
+                logger.exception("Proxy relay error after %d bytes", total)
+            finally:
+                logger.info("Proxy relay done: %d bytes", total)
+                await resp.aclose()
+                await client.aclose()
+
+        return StreamingResponse(relay(), media_type="application/octet-stream")
+
+    # ── Gradio UI ────────────────────────────────────────────────
+    css = """
+    #generate-btn button { width: 100%; }
+    #streaming-player { border: 1px solid var(--border-color-primary) !important; border-radius: var(--block-radius) !important; padding: var(--block-padding) !important; }
+    """
+
+    theme = gr.themes.Default(
+        primary_hue=gr.themes.Color(
+            c50="#E8F5E9",
+            c100="#C8E6C9",
+            c200="#A5D6A7",
+            c300="#81C784",
+            c400="#66BB6A",
+            c500="#4CAF50",
+            c600="#43A047",
+            c700="#388E3C",
+            c800="#2E7D32",
+            c900="#1B5E20",
+            c950="#0D3B10",
+        ),
+    )
+
+    with gr.Blocks(title="MOSS-TTS-Nano Demo") as demo:
+        gr.HTML(f"""
+        <div style="display:flex; align-items:center; gap:16px; margin-bottom:8px;">
+          <img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
+               alt="vLLM-Omni" style="height:42px;">
+          <div>
+            <h1 style="margin:0; font-size:1.5em;">MOSS-TTS-Nano Streaming Demo</h1>
+            <span style="font-size:0.85em; color:#666;">
+              0.1B AR LM + MOSS-Audio-Tokenizer-Nano &middot; 48 kHz &middot; ZH / EN / JA
+              &nbsp;&middot;&nbsp; Served by
+              <a href="https://github.com/vllm-project/vllm-omni" target="_blank"
+              style="color:#43A047; text-decoration:none; font-weight:600;">vLLM-Omni</a>
+              &nbsp;&middot;&nbsp; <code style="background:#eef2f7; padding:2px 6px; border-radius:4px; font-size:0.9em;">{api_base}</code>
+            </span>
+          </div>
+        </div>
+        """)
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                text_input = gr.Textbox(
+                    label="Text to Synthesize",
+                    placeholder="Enter text in Chinese, English, or Japanese...",
+                    lines=4,
+                )
+
+                voice_preset = gr.Dropdown(
+                    choices=PRESET_LABELS,
+                    value=DEFAULT_PRESET,
+                    label="Voice Preset",
+                    info="Built-in multilingual voices. ZH = Chinese, EN = English, JA = Japanese.",
+                )
+
+                with gr.Accordion("Custom Voice Clone (optional)", open=False):
+                    gr.Markdown(
+                        "Upload a reference audio clip (10-30 s) and provide its transcript "
+                        "to clone that voice instead of using the built-in preset."
+                    )
+                    ref_audio = gr.Audio(
+                        label="Reference Audio",
+                        type="numpy",
+                        sources=["upload", "microphone"],
+                    )
+                    ref_text = gr.Textbox(
+                        label="Reference Audio Transcript (required for cloning)",
+                        placeholder="Exact transcript of the reference audio...",
+                        lines=2,
+                    )
+
+                with gr.Row():
+                    response_format = gr.Dropdown(
+                        choices=["wav", "mp3", "flac", "pcm"],
+                        value="pcm",
+                        label="Audio Format",
+                        interactive=False,
+                        scale=1,
+                    )
+                    stream_checkbox = gr.Checkbox(
+                        label="Stream (gapless)",
+                        value=True,
+                        info="AudioWorklet streaming (recommended)",
+                        scale=1,
+                    )
+
+                with gr.Row():
+                    generate_btn = gr.Button(
+                        "Generate Speech",
+                        variant="primary",
+                        size="lg",
+                        elem_id="generate-btn",
+                        scale=3,
+                    )
+                    reset_btn = gr.Button(
+                        "Reset",
+                        variant="secondary",
+                        size="lg",
+                        scale=1,
+                    )
+
+            with gr.Column(scale=2):
+                player_html = gr.HTML(
+                    value=PLAYER_HTML,
+                    visible=True,
+                    label="streaming player",
+                    elem_id="streaming-player",
+                )
+                audio_output = gr.Audio(
+                    label="Generated Audio",
+                    interactive=False,
+                    autoplay=True,
+                    visible=False,
+                )
+                gr.Examples(
+                    examples=[
+                        ["Hello, this is MOSS-TTS-Nano speaking.", "Ava (EN)"],
+                        ["The quick brown fox jumps over the lazy dog.", "Adam (EN)"],
+                        ["MOSS-TTS-Nanoは、軽量なテキスト読み上げモデルです。", "Sakura (JA)"],
+                    ],
+                    inputs=[text_input, voice_preset],
+                    label="Examples",
+                )
+                gr.HTML("""
+                <div style="text-align:center; padding:8px 0; margin-top:4px;">
+                  <a href="https://github.com/vllm-project/vllm-omni" target="_blank">
+                    <img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
+                         alt="vLLM-Omni" style="height:28px; opacity:0.7;">
+                  </a>
+                </div>
+                """)
+
+        # Hidden textbox to pass payload from Python -> JavaScript
+        hidden_payload = gr.Textbox(visible=False, elem_id="tts-payload")
+
+        # ── Event wiring ─────────────────────────────────────────
+        def on_stream_change(stream: bool):
+            if stream:
+                return (
+                    gr.update(value="pcm", interactive=False),
+                    gr.update(visible=True),  # player
+                    gr.update(visible=False),  # audio
+                )
+            return (
+                gr.update(value="wav", interactive=True),
+                gr.update(visible=False),
+                gr.update(visible=True),
+            )
+
+        stream_checkbox.change(
+            fn=on_stream_change,
+            inputs=[stream_checkbox],
+            outputs=[response_format, player_html, audio_output],
+        )
+
+        def on_reset():
+            return (
+                "",  # text
+                None,  # audio_output
+                "",  # hidden_payload
+                PLAYER_HTML,  # reset player
+            )
+
+        reset_btn.click(
+            fn=on_reset,
+            outputs=[text_input, audio_output, hidden_payload, player_html],
+            js="() => { if (window.ttsStop) window.ttsStop(); }",
+        )
+
+        all_inputs = [text_input, voice_preset, ref_audio, ref_text, response_format]
+
+        def on_generate(stream_enabled, *args):
+            import time as _time
+
+            text, voice, ref_a, ref_t, _fmt = args
+            if stream_enabled:
+                payload = build_payload(text, voice, ref_a, ref_t, stream=True)
+                payload["_nonce"] = int(_time.time() * 1000)
+                return json.dumps(payload), gr.update()
+            else:
+                audio = generate_speech(api_base, text, voice, ref_a, ref_t, _fmt)
+                return "", audio
+
+        generate_btn.click(
+            fn=on_generate,
+            inputs=[stream_checkbox] + all_inputs,
+            outputs=[hidden_payload, audio_output],
+        ).then(
+            fn=lambda p: p,
+            inputs=[hidden_payload],
+            outputs=[hidden_payload],
+            js="(p) => { if (p && p.trim()) { const d = JSON.parse(p); delete d._nonce; window.ttsGenerate(d); } return p; }",
+        )
+
+        demo.queue()
+
+    return gr.mount_gradio_app(
+        fastapi_app,
+        demo,
+        path="/",
+        css=css,
+        theme=theme,
+        head=_build_player_js(PCM_SAMPLE_RATE),
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Gradio demo for MOSS-TTS-Nano with gapless AudioWorklet streaming.",
+    )
+    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
+    parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)")
+    parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)")
+    parser.add_argument("--share", action="store_true", help="Share publicly via Gradio tunnel")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+    print(f"Connecting to vLLM-Omni server at: {args.api_base}")
+
+    app = create_app(args.api_base)
+
+    import uvicorn
+
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/moss_tts_nano/run_gradio_demo.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Launch MOSS-TTS-Nano server + Gradio demo together.
+#
+# Usage:
+#   ./run_gradio_demo.sh
+#   CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh
+
+set -e
+
+MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}"
+PORT="${PORT:-8091}"
+GRADIO_PORT="${GRADIO_PORT:-7860}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+echo "Starting MOSS-TTS-Nano server (port $PORT)..."
+FLASHINFER_DISABLE_VERSION_CHECK=1 \
+vllm serve "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --omni &
+SERVER_PID=$!
+
+cleanup() {
+    echo "Stopping server (PID $SERVER_PID)..."
+    kill $SERVER_PID 2>/dev/null
+    wait $SERVER_PID 2>/dev/null
+}
+trap cleanup EXIT
+
+# Wait for server to be ready.
+echo "Waiting for server to start..."
+for i in $(seq 1 120); do
+    if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
+        echo "Server ready."
+        break
+    fi
+    sleep 2
+done
+
+echo "Starting Gradio demo (port $GRADIO_PORT)..."
+python "$SCRIPT_DIR/gradio_demo.py" \
+    --api-base "http://localhost:$PORT" \
+    --port "$GRADIO_PORT"

examples/online_serving/moss_tts_nano/run_server.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Launch vLLM-Omni server for MOSS-TTS-Nano
+#
+# Usage:
+#   ./run_server.sh
+#   CUDA_VISIBLE_DEVICES=0 PORT=8091 ./run_server.sh
+
+set -e
+
+MODEL="${MODEL:-OpenMOSS-Team/MOSS-TTS-Nano}"
+PORT="${PORT:-8091}"
+
+echo "Starting MOSS-TTS-Nano server with model: $MODEL"
+
+FLASHINFER_DISABLE_VERSION_CHECK=1 \
+vllm serve "$MODEL" \
+    --host 0.0.0.0 \
+    --port "$PORT" \
+    --omni

examples/online_serving/qwen3_omni/README.md

@@ -31,11 +31,13 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
 ### Launch individual stages (stage-based CLI)
 
 Use the stage-based CLI when you want to run one stage per process.
+The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via
+`CUDA_VISIBLE_DEVICES`.
 
 **1. Stage 0 (Thinker + API server)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
@@ -45,7 +47,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 **2. Stage 1 (Talker)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -55,7 +57,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 **3. Stage 2 (Code2Wav)**
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 2 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -78,7 +80,7 @@ for this category of optimization. Given that each instantiation strictly initia
 parameter flags can be systematically assigned directly onto that specific stage's command sequence:
 
 ```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \

recipes/Qwen/Qwen3-Omni.md

@@ -1,76 +1,141 @@
-# Qwen3-Omni for multimodal chat on 1x A100 80GB
+# Qwen3-Omni
 
 ## Summary
 
 - Vendor: Qwen
 - Model: `Qwen/Qwen3-Omni-30B-A3B-Instruct`
-- Task: Multimodal chat with text, image, audio, or video input
+- Task: Multimodal chat with text/image/audio/video input
 - Mode: Online serving with the OpenAI-compatible API
 - Maintainer: Community
 
 ## When to use this recipe
 
-Use this recipe when you want a known-good starting point for serving
-`Qwen/Qwen3-Omni-30B-A3B-Instruct` with vLLM-Omni on a single 80 GB A100 and
-validate the deployment with the existing multimodal client examples in this
-repository.
+Use this recipe as a practical baseline for running
+`Qwen/Qwen3-Omni-30B-A3B-Instruct` with the same serving paths already
+exercised by repository examples and tests.
 
 ## References
 
-- Upstream or canonical docs:
+- User guide:
   [`docs/user_guide/examples/online_serving/qwen3_omni.md`](../../docs/user_guide/examples/online_serving/qwen3_omni.md)
-- Related example under `examples/`:
+- Example guide:
   [`examples/online_serving/qwen3_omni/README.md`](../../examples/online_serving/qwen3_omni/README.md)
-- Related issue or discussion:
-  [RFC: add recipes folder](https://github.com/vllm-project/vllm-omni/issues/2645)
 
-## Hardware Support
-
-This recipe currently documents one tested-style reference configuration for
-CUDA GPU serving. Add more sections for other hardware as community validation
-lands.
-
-## GPU
-
-### 1x A100 80GB
-
-#### Environment
+## Environment
 
 - OS: Linux
 - Python: 3.10+
-- Driver / runtime: NVIDIA CUDA environment with an A100 80 GB GPU
-- vLLM version: Match the repository requirements for your checkout
-- vLLM-Omni version or commit: Use the commit you are deploying from
+- vLLM / vLLM-Omni: use versions from your current checkout, >=0.18.0
 
-#### Command
+## Start server (single command)
 
-Start the server from the repository root:
+From repository root:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
-Async chunking is enabled by default in the bundled deployment config. For
-common runtime tuning, prefer CLI overrides instead of editing or passing a
-custom YAML file:
+Notes:
 
-```bash
-# Disable async chunking for /v1/realtime sessions
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
-  --no-async-chunk
-```
+- `--omni` is required.
+- The default deploy config `vllm_omni/deploy/qwen3_omni_moe.yaml` is loaded
+  automatically by model registry.
+- `async_chunk` is enabled by default in this deploy config.
+- Platform deltas under `platforms:` (NPU/ROCm/XPU) are merged automatically on
+  matching runtimes.
 
-Use a custom deploy config only for advanced cases such as custom topology,
-connector wiring, or a larger overlay of stage defaults:
+For advanced customization, pass an overlay YAML:
 
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
   --deploy-config /path/to/your_qwen3_omni_overrides.yaml
 ```
 
+#### Runtime tuning
+
+Prefer CLI overrides for day-to-day tuning:
+
+```bash
+# Disable async chunking when using /v1/realtime
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+  --no-async-chunk
+
+# Example per-stage tuning in unified launch
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+  --stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
+
+# Tune max_num_seqs per stage (single process launch)
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
+  --stage-overrides '{
+    "0": {"max_num_seqs": 8},
+    "1": {"max_num_seqs": 4},
+    "2": {"max_num_seqs": 4}
+  }'
+```
+
+#### Stage-based launch (one stage per process)
+
+Use three terminals (one per stage). Start with the default commands below, then
+add `--max-num-seqs` only if you need explicit per-stage concurrency control.
+
+Default stage-based commands:
+
+```bash
+# Stage 0: Thinker + API server
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --port 8091 \
+  --stage-id 0 \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+
+# Stage 1: Talker
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --stage-id 1 \
+  --headless \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+
+# Stage 2: Code2Wav
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --stage-id 2 \
+  --headless \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+```
+
+Optional: explicit per-stage `max_num_seqs` tuning:
+
+```bash
+# Stage 0
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --port 8091 \
+  --stage-id 0 \
+  --max-num-seqs 8 \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+
+# Stage 1
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --stage-id 1 \
+  --headless \
+  --max-num-seqs 4 \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+
+# Stage 2
+CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+  --stage-id 2 \
+  --headless \
+  --max-num-seqs 4 \
+  --omni-master-address 127.0.0.1 \
+  --omni-master-port 26000 &
+```
+
+If you use custom deploy YAML, add `--deploy-config` to each stage command.
+
 #### Verification
 
-Run one of the existing example clients after the server is ready:
+After server startup, run a multimodal example client:
 
 ```bash
 python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py \
@@ -80,7 +145,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
   --host localhost
 ```
 
-For a quick API smoke test, request text-only output:
+Quick API smoke test (text-only output):
 
 ```bash
 curl http://localhost:8091/v1/chat/completions \
@@ -92,8 +157,89 @@ curl http://localhost:8091/v1/chat/completions \
   }'
 ```
 
+`{"modalities":["text","audio"]}` means the model returns both text and audio in
+the same response. Use it when you want transcription/content text and TTS audio
+together.
+
+Quick API smoke test (text + audio output):
+
+```bash
+curl http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+    "messages": [{"role": "user", "content": "Describe vLLM in brief."}],
+    "modalities": ["text", "audio"]
+  }'
+```
+
+Realtime WebSocket check (`/v1/realtime`) requires async chunk disabled:
+
+```bash
+python examples/online_serving/qwen3_omni/openai_realtime_client.py \
+  --url ws://localhost:8091/v1/realtime \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --input-wav /path/to/input_16k_mono.wav \
+  --output-wav realtime_output.wav
+```
+
+#### Benchmark with `vllm bench`
+
+After the server is up, you can run online serving benchmarks with
+`vllm bench serve --omni`.
+
+Text-focused random workload:
+
+```bash
+vllm bench serve \
+  --omni \
+  --host localhost \
+  --port 8091 \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --endpoint /v1/chat/completions \
+  --backend openai-chat-omni \
+  --dataset-name random \
+  --num-prompts 40 \
+  --max-concurrency 4 \
+  --random-input-len 2500 \
+  --random-output-len 900 \
+  --ignore-eos \
+  --extra-body '{"modalities":["text"]}' \
+  --percentile-metrics ttft,tpot,itl,e2el
+```
+
+If you want benchmark requests to return both text and audio, switch
+`--extra-body` to:
+
+```bash
+--extra-body '{"modalities":["text","audio"]}'
+```
+
+Synthetic multimodal workload (`random-mm`):
+
+```bash
+vllm bench serve \
+  --omni \
+  --host localhost \
+  --port 8091 \
+  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+  --endpoint /v1/chat/completions \
+  --backend openai-chat-omni \
+  --dataset-name random-mm \
+  --num-prompts 40 \
+  --max-concurrency 4 \
+  --random-input-len 2500 \
+  --random-output-len 900 \
+  --random-mm-base-items-per-request 2 \
+  --random-mm-limit-mm-per-prompt '{"image":1,"video":1,"audio":1}' \
+  --random-mm-bucket-config '{"(32, 32, 1)": 0.5, "(0, 1, 1)": 0.5}' \
+  --ignore-eos \
+  --extra-body '{"modalities":["text"]}' \
+  --percentile-metrics ttft,tpot,itl,e2el
+```
+
 #### Notes
 
-- Memory usage: Size depends on runtime options and output modalities; leave headroom for multimodal workloads. Prefer CLI overrides such as `--gpu-memory-utilization` for routine tuning.
-- Key flags: `--omni` is required; async chunking is enabled by default; use `--no-async-chunk` for realtime sessions and `--deploy-config` only for advanced custom deployments.
-- Known limitations: The `/v1/realtime` WebSocket flow is currently unsupported while async chunking is enabled. This starter recipe is intentionally narrow and focuses on the single-GPU online-serving path already documented in the repo examples.
+- `/v1/realtime` is unsupported while `async_chunk` is enabled.
+- The default deploy uses `SharedMemoryConnector`; this is for single-host
+  stage wiring.

requirements/musa.txt

@@ -1,6 +1,6 @@
 -r common.txt
 # MUSA platform dependencies
-torchada>=0.1.50
+torchada>=0.1.52
 onnxruntime>=1.23.2
 mate>=0.2.0
 flash_attn_3>=0.1.4

tests/benchmarks/conftest.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""conftest.py for benchmarks unit tests.
+
+Installs lightweight mock stubs for ``vllm`` (and sub-packages) so the
+data-module unit tests can run without a full vLLM installation.  Only the
+symbols actually imported by
+``vllm_omni.benchmarks.data_modules.seed_tts_dataset`` are emulated.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from dataclasses import dataclass
+from typing import Any
+
+
+def _install_vllm_stubs() -> None:
+    """Register minimal vllm stubs in sys.modules.
+
+    Only installs when real vllm is unavailable.  We actively probe the
+    import because an empty or partial vllm may not yet have imported
+    the submodules we rely on, and unconditionally registering stubs
+    would shadow the real package for sibling tests (e.g.
+    ``tests/benchmarks/metrics/test_metrics.py`` needs the real
+    ``vllm.benchmarks.serve``).
+    """
+    try:
+        import vllm.benchmarks.datasets  # noqa: F401
+        import vllm.tokenizers  # noqa: F401
+    except ImportError:
+        pass
+    else:
+        return  # real vllm available — do not shadow it
+    if "vllm.benchmarks.datasets" in sys.modules:
+        return
+
+    # ------------------------------------------------------------------ #
+    # vllm.benchmarks.datasets                                            #
+    # ------------------------------------------------------------------ #
+    @dataclass
+    class SampleRequest:
+        prompt: str = ""
+        prompt_len: int = 0
+        expected_output_len: int = 0
+        multi_modal_data: Any = None
+        request_id: str = ""
+
+    class BenchmarkDataset:
+        def __init__(
+            self,
+            dataset_path: str = "",
+            random_seed: int = 0,
+            disable_shuffle: bool = False,
+            **kwargs: Any,
+        ) -> None:
+            self.dataset_path = dataset_path
+            self.random_seed = random_seed
+            self.disable_shuffle = disable_shuffle
+
+        def maybe_oversample_requests(
+            self,
+            out: list,
+            num_requests: int,
+            request_id_prefix: str,
+            no_oversample: bool,
+        ) -> None:
+            pass
+
+    # ------------------------------------------------------------------ #
+    # vllm.tokenizers / vllm.tokenizers.hf                               #
+    # ------------------------------------------------------------------ #
+    class TokenizerLike:
+        pass
+
+    def get_cached_tokenizer(t: Any) -> Any:
+        return t
+
+    # ------------------------------------------------------------------ #
+    # Wire up sys.modules                                                 #
+    # ------------------------------------------------------------------ #
+    vllm_mod = types.ModuleType("vllm")
+    vllm_benchmarks = types.ModuleType("vllm.benchmarks")
+    vllm_benchmarks_datasets = types.ModuleType("vllm.benchmarks.datasets")
+    vllm_tokenizers = types.ModuleType("vllm.tokenizers")
+    vllm_tokenizers_hf = types.ModuleType("vllm.tokenizers.hf")
+
+    vllm_benchmarks_datasets.BenchmarkDataset = BenchmarkDataset  # type: ignore[attr-defined]
+    vllm_benchmarks_datasets.SampleRequest = SampleRequest  # type: ignore[attr-defined]
+    vllm_tokenizers.TokenizerLike = TokenizerLike  # type: ignore[attr-defined]
+    vllm_tokenizers_hf.get_cached_tokenizer = get_cached_tokenizer  # type: ignore[attr-defined]
+
+    sys.modules["vllm"] = vllm_mod
+    sys.modules["vllm.benchmarks"] = vllm_benchmarks
+    sys.modules["vllm.benchmarks.datasets"] = vllm_benchmarks_datasets
+    sys.modules["vllm.tokenizers"] = vllm_tokenizers
+    sys.modules["vllm.tokenizers.hf"] = vllm_tokenizers_hf
+
+
+# Install stubs immediately at collection time (before any test import).
+_install_vllm_stubs()

tests/benchmarks/test_bench_tts_cli.py

@@ -0,0 +1,139 @@
+"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+# Add benchmarks/tts to path for import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "benchmarks" / "tts"))
+import bench_tts
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+@pytest.fixture()
+def model_configs_path(tmp_path: Path) -> Path:
+    cfg = {
+        "models": {
+            "test/ModelA": {
+                "stage_config": "model_a.yaml",
+                "supported_tasks": ["voice_clone", "default_voice"],
+                "backend": "openai-audio-speech",
+                "endpoint": "/v1/audio/speech",
+                "task_extra_body": {
+                    "voice_clone": {"task_type": "Base"},
+                    "default_voice": {"voice": "Vivian", "task_type": "CustomVoice"},
+                },
+            },
+            "test/ModelB": {
+                "stage_config": "model_b.yaml",
+                "supported_tasks": ["voice_clone"],
+                "backend": "openai-audio-speech",
+                "endpoint": "/v1/audio/speech",
+                "task_extra_body": {"voice_clone": {}},
+            },
+        }
+    }
+    p = tmp_path / "model_configs.yaml"
+    p.write_text(yaml.dump(cfg), encoding="utf-8")
+    return p
+
+
+def test_load_model_configs(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    assert "test/ModelA" in configs
+    assert "test/ModelB" in configs
+    assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
+
+
+def test_build_bench_args_voice_clone(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="voice_clone",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=False,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    assert "--dataset-name" in cmd
+    idx = cmd.index("--dataset-name")
+    assert cmd[idx + 1] == "seed-tts"
+    assert "--max-concurrency" in cmd
+    assert "--extra-body" in cmd
+    extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
+    assert extra_body.get("task_type") == "Base"
+
+
+def test_build_bench_args_default_voice_has_voice_param(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="default_voice",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=False,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    idx = cmd.index("--dataset-name")
+    assert cmd[idx + 1] == "seed-tts-text"
+    extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
+    assert extra_body.get("voice") == "Vivian"
+
+
+def test_build_bench_args_wer_eval_adds_flag(model_configs_path: Path) -> None:
+    configs = bench_tts.load_model_configs(model_configs_path)
+    cmd = bench_tts.build_bench_args(
+        host="localhost",
+        port=8000,
+        model="test/ModelA",
+        task="voice_clone",
+        model_cfg=configs["test/ModelA"],
+        locale="en",
+        num_prompts=10,
+        concurrency=1,
+        dataset_path="/data/seed-tts",
+        wer_eval=True,
+        output_dir=None,
+        result_filename=None,
+        extra_cli_args=[],
+    )
+    assert "--seed-tts-wer-eval" in cmd
+
+
+def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.CaptureFixture, mocker) -> None:
+    # ModelB does not support voice_design
+    mocker.patch.object(
+        sys,
+        "argv",
+        [
+            "bench_tts.py",
+            "--model",
+            "test/ModelB",
+            "--task",
+            "voice_design",
+            "--model-configs",
+            str(model_configs_path),
+        ],
+    )
+    with pytest.raises(SystemExit):
+        bench_tts.main()

tests/benchmarks/test_seed_tts_dataset_variants.py

@@ -0,0 +1,169 @@
+"""Tests for SeedTTSTextDataset, SeedTTSTextSampleRequest, SeedTTSDesignDataset,
+and SeedTTSDesignSampleRequest.
+
+vllm stubs are installed by tests/benchmarks/conftest.py before collection.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "seed_tts_dataset.py"
+_MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset"
+
+if _MODULE_NAME not in sys.modules:
+    _spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
+    _mod = importlib.util.module_from_spec(_spec)
+    sys.modules[_MODULE_NAME] = _mod
+    _spec.loader.exec_module(_mod)
+
+from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (  # noqa: E402
+    SeedTTSDesignDataset,
+    SeedTTSDesignSampleRequest,
+    SeedTTSTextDataset,
+    SeedTTSTextSampleRequest,
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def seed_tts_root(tmp_path: Path) -> Path:
+    """Minimal seed-tts-style directory with 5 entries."""
+    locale_dir = tmp_path / "en"
+    locale_dir.mkdir()
+    wav_dir = locale_dir / "prompt-wavs"
+    wav_dir.mkdir()
+    for i in range(5):
+        (wav_dir / f"utt{i:03d}.wav").write_bytes(b"RIFF\x00\x00\x00\x00WAVE")
+    meta = "\n".join(f"utt{i:03d}|ref text {i}|prompt-wavs/utt{i:03d}.wav|target text {i}" for i in range(5))
+    (locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
+    return tmp_path
+
+
+@pytest.fixture()
+def mock_tokenizer(mocker):
+    tokenizer = mocker.MagicMock()
+    tokenizer.encode = lambda text, **kw: [0] * len(text.split())
+    tokenizer.get_vocab.return_value = {"<pad>": 0}
+    tokenizer.all_special_ids = []
+    tokenizer.all_special_tokens = []
+    tokenizer.vocab_size = 1
+    tokenizer.__len__.return_value = 1
+    return tokenizer
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
+    ds = SeedTTSTextDataset(
+        dataset_path=str(seed_tts_root),
+        random_seed=0,
+        locale="en",
+        disable_shuffle=True,
+    )
+    requests = ds.sample(mock_tokenizer, num_requests=3)
+    assert len(requests) == 3
+    for req in requests:
+        assert isinstance(req, SeedTTSTextSampleRequest)
+        assert req.seed_tts_speech_extra is None or "ref_audio" not in (req.seed_tts_speech_extra or {})
+        assert req.seed_tts_ref_wav_path == ""
+        assert "target text" in req.prompt
+
+
+# ---------------------------------------------------------------------------
+# SeedTTSDesignDataset tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def seed_tts_design_root(tmp_path: Path) -> Path:
+    """seed-tts-design directory with 5-field meta.lst entries."""
+    locale_dir = tmp_path / "en"
+    locale_dir.mkdir()
+    meta = "\n".join(
+        f"des{i:03d}|||target text {i}|A warm {['female', 'male'][i % 2]} voice with neutral accent." for i in range(5)
+    )
+    (locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
+    return tmp_path
+
+
+def test_seed_tts_design_dataset_has_instructions(seed_tts_design_root, mock_tokenizer):
+    ds = SeedTTSDesignDataset(
+        dataset_path=str(seed_tts_design_root),
+        random_seed=0,
+        locale="en",
+        disable_shuffle=True,
+    )
+    requests = ds.sample(mock_tokenizer, num_requests=3)
+    assert len(requests) == 3
+    for req in requests:
+        assert isinstance(req, SeedTTSDesignSampleRequest)
+        extra = req.seed_tts_speech_extra or {}
+        assert "instructions" in extra
+        assert extra["instructions"], "instructions must be non-empty"
+        assert extra.get("task_type") == "VoiceDesign"
+        assert "ref_audio" not in extra
+        assert req.seed_tts_ref_wav_path == ""
+
+
+def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_root, mock_tokenizer):
+    """Lines without a voice_description should be skipped."""
+    locale_dir = seed_tts_design_root / "en"
+    # The bad line has 4 fields, not 5, so will be filtered
+    meta = "bad|||target text without description\n" + "\n".join(
+        f"ok|||target text {i}|A clear female voice." for i in range(9)
+    )
+    (locale_dir / "meta.lst").write_text(meta, encoding="utf-8")
+    ds = SeedTTSDesignDataset(
+        dataset_path=str(seed_tts_design_root),
+        random_seed=0,
+        locale="en",
+        disable_shuffle=True,
+    )
+    requests = ds.sample(mock_tokenizer, num_requests=10, no_oversample=True)
+    assert len(requests) == 9  # since we filter the bad row out and don't oversample
+    for req in requests:
+        assert isinstance(req, SeedTTSDesignSampleRequest)
+        assert req.seed_tts_utterance_id == "ok"
+
+
+def test_attach_sets_seed_tts_row_even_without_extra_body():
+    """seed_tts_row=True must be set for SeedTTSTextSampleRequest (no extra body)."""
+    from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSTextSampleRequest
+
+    req = SeedTTSTextSampleRequest(
+        prompt="hello world",
+        prompt_len=2,
+        expected_output_len=100,
+        multi_modal_data=None,
+        request_id="test-0",
+        seed_tts_speech_extra=None,
+        seed_tts_ref_wav_path="",
+    )
+    assert req.seed_tts_speech_extra is None
+    assert req.seed_tts_ref_wav_path == ""
+    # The fix ensures that even with speech_extra=None, the function
+    # sets seed_tts_row=True. We verify the source code has the fix.
+    import inspect
+
+    import vllm_omni.benchmarks.patch.patch as patch_mod
+
+    src = inspect.getsource(patch_mod._attach_seed_tts_to_request_func_input)
+    # seed_tts_row must be set BEFORE the 'if not ex: return' check
+    row_pos = src.index("seed_tts_row")
+    not_ex_pos = src.index("if not ex:")
+    assert row_pos < not_ex_pos, "seed_tts_row must be set before 'if not ex: return'"

tests/config/test_pipeline_registry.py

@@ -6,11 +6,7 @@ from __future__ import annotations
 
 import pytest
 
-from vllm_omni.config.pipeline_registry import (
-    _DIFFUSION_PIPELINES,
-    _OMNI_PIPELINES,
-    _VLLM_OMNI_PIPELINES,
-)
+from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
 from vllm_omni.config.stage_config import (
     _PIPELINE_REGISTRY,
     PipelineConfig,
@@ -23,17 +19,9 @@ from vllm_omni.config.stage_config import (
 class TestCentralRegistryDeclarations:
     """Every in-tree pipeline must be declared exactly once in the central registry."""
 
-    def test_union_contains_all_omni(self):
+    def test_omni_entries_visible_in_registry(self):
         for key in _OMNI_PIPELINES:
-            assert key in _VLLM_OMNI_PIPELINES
-
-    def test_union_contains_all_diffusion(self):
-        for key in _DIFFUSION_PIPELINES:
-            assert key in _VLLM_OMNI_PIPELINES
-
-    def test_no_duplicate_model_type_between_omni_and_diffusion(self):
-        overlap = set(_OMNI_PIPELINES) & set(_DIFFUSION_PIPELINES)
-        assert not overlap, f"Duplicate model_types across omni/diffusion: {overlap}"
+            assert key in _PIPELINE_REGISTRY
 
     def test_expected_omni_pipelines_present(self):
         # Guard against accidental removal during future refactors.

tests/dfx/conftest.py

@@ -1,13 +1,18 @@
 import json
 import os
+import re
 import subprocess
+import time
 from datetime import datetime
 from pathlib import Path
 from typing import Any
 
 import pytest
 
+from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw
+from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import modify_stage_config
+from vllm_omni.platforms import current_omni_platform
 
 
 def load_configs(config_path: str) -> list[dict[str, Any]]:
@@ -26,18 +31,15 @@ def load_configs(config_path: str) -> list[dict[str, Any]]:
         raise RuntimeError(f"Failed to load configuration file: {str(e)}")
 
 
-def modify_stage(default_path, updates, deletes):
-    kwargs = {}
+def modify_stage(default_path: str, updates: dict[str, Any] | None, deletes: dict[str, Any] | None) -> str:
+    kwargs: dict[str, Any] = {}
     if updates is not None:
         kwargs["updates"] = updates
     if deletes is not None:
         kwargs["deletes"] = deletes
     if kwargs:
-        path = modify_stage_config(default_path, **kwargs)
-    else:
-        path = default_path
-
-    return path
+        return modify_stage_config(default_path, **kwargs)
+    return default_path
 
 
 def _build_serve_args(serve_args: Any) -> list[str]:
@@ -111,6 +113,122 @@ def create_unique_server_params(
     return unique_params
 
 
+def configs_with_platform_stage_configs(configs: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Resolve stage config for XPU vs CUDA/ROCm."""
+    out: list[dict[str, Any]] = []
+    for config in configs:
+        config_copy = json.loads(json.dumps(config))
+        if current_omni_platform.is_xpu():
+            config_copy["server_params"]["stage_config_name"] = "xpu/qwen3_omni_ci.yaml"
+        out.append(config_copy)
+    return out
+
+
+def extract_server_args_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[str] | None]:
+    mapping: dict[str, list[str] | None] = {}
+    for cfg in configs:
+        test_name = str(cfg.get("test_name"))
+        server_params = cfg.get("server_params") or {}
+        raw_args = server_params.get("server_args")
+        mapping[test_name] = [str(item) for item in raw_args] if isinstance(raw_args, list) else None
+    return mapping
+
+
+def create_reliability_omni_server_params(
+    configs: list[dict[str, Any]], stage_configs_dir: Path
+) -> list[OmniServerParams]:
+    adjusted_configs = configs_with_platform_stage_configs(configs)
+    unique_params = create_unique_server_params(adjusted_configs, stage_configs_dir)
+    server_args_by_name = extract_server_args_by_test_name(adjusted_configs)
+    return [
+        OmniServerParams(
+            model=model,
+            stage_config_path=stage_config_path,
+            server_args=server_args_by_name.get(test_name),
+        )
+        for test_name, model, stage_config_path, _stage_overrides_json, _extra_cli_args in unique_params
+    ]
+
+
+def supports_video_generation(model_name: str) -> bool:
+    lower = model_name.lower()
+    return any(key in lower for key in ("wan", "video", "i2v", "t2v"))
+
+
+def supports_chat_generation(model_name: str) -> bool:
+    return not supports_video_generation(model_name)
+
+
+def parse_stage_devices(stage_config_path: str) -> str:
+    text = Path(stage_config_path).read_text(encoding="utf-8")
+    raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE)
+    devices: set[int] = set()
+    for item in raw_devices:
+        for token in item.split(","):
+            token = token.strip()
+            if token:
+                devices.add(int(token))
+    if not devices:
+        raise ValueError(f"No runtime.devices found in stage config: {stage_config_path}")
+    return ",".join(str(x) for x in sorted(devices))
+
+
+def resolve_oom_device_spec(config: dict[str, Any], stage_config_path: str | None) -> str:
+    explicit = config.get("device")
+    if explicit is not None:
+        return str(explicit)
+    if not stage_config_path:
+        return "0"
+    return parse_stage_devices(stage_config_path)
+
+
+def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> None:
+    text = str(exc).lower()
+    assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}"
+
+
+def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None:
+    """Poll a minimal chat request until success."""
+    deadline = time.time() + timeout_sec
+    payload = json.dumps(
+        {
+            "model": model,
+            "messages": [{"role": "user", "content": "Say hello in one short sentence."}],
+            "stream": False,
+            "modalities": ["text"],
+        }
+    )
+    last_error: str | None = None
+    while time.time() < deadline:
+        try:
+            status, body = post_chat_completions_raw(host, port, payload)
+            if status == 200:
+                return
+            last_error = f"http={status} body={body[:200]!r}"
+        except Exception as exc:  # noqa: BLE001
+            last_error = str(exc)
+        time.sleep(2)
+    raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}")
+
+
+def assert_no_extra_worker_processes(
+    baseline_pids: set[int],
+    worker_pattern: str,
+    timeout_sec: int = 60,
+) -> None:
+    """Ensure no extra worker PIDs remain after teardown."""
+    deadline = time.time() + timeout_sec
+    while time.time() < deadline:
+        current = set(list_remote_process_pids_by_pattern(worker_pattern))
+        extra = current - baseline_pids
+        if not extra:
+            return
+        time.sleep(2)
+    current = set(list_remote_process_pids_by_pattern(worker_pattern))
+    extra = sorted(current - baseline_pids)
+    assert not extra, f"orphan worker processes remain after container teardown: {extra}"
+
+
 def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]:
     mapping = {}
     for config in configs:
@@ -120,7 +238,11 @@ def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, di
                 "test_name": test_name,
                 "benchmark_params": [],
             }
-        mapping[test_name]["benchmark_params"].extend(config["benchmark_params"])
+        for entry in config["benchmark_params"]:
+            # Skip disabled entries
+            if not entry.get("enabled", True):
+                continue
+            mapping[test_name]["benchmark_params"].append(entry)
     return mapping
 
 

tests/dfx/perf/scripts/run_benchmark.py

@@ -333,7 +333,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         raise ValueError("The number of prompts does not match the QPS or max_concurrency")
 
     args = ["--host", host, "--port", str(port)]
-    exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency"}
+    exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
 
     for key, value in params.items():
         if key in exclude_keys or value is None:

tests/dfx/perf/tests/test_runner_metadata.py

@@ -0,0 +1,79 @@
+"""Tests for DFX runner metadata field exclusion."""
+
+import json
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_task_excluded_from_cli_args():
+    """'task' field must not become --task CLI arg."""
+    params = {
+        "task": "voice_clone",
+        "dataset_name": "seed-tts",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "percentile-metrics": "audio_rtf,audio_ttfp",
+        "baseline": {"mean_audio_rtf": [0.5]},
+    }
+    exclude_keys = {"request_rate", "baseline", "num_prompts", "max_concurrency", "task", "enabled", "eval_phase"}
+    args = []
+    for key, value in params.items():
+        if key in exclude_keys or value is None:
+            continue
+        arg_name = f"--{key.replace('_', '-')}"
+        if isinstance(value, bool) and value:
+            args.append(arg_name)
+        elif isinstance(value, dict):
+            args.extend([arg_name, json.dumps(value)])
+        elif not isinstance(value, bool):
+            args.extend([arg_name, str(value)])
+    assert "--task" not in args
+    assert "--enabled" not in args
+    assert "--dataset-name" in args
+
+
+def test_enabled_false_entry_is_skipped():
+    """benchmark_params entry with enabled=false should be skipped."""
+    import sys
+    from pathlib import Path
+
+    sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent))
+    from tests.dfx.conftest import create_test_parameter_mapping
+
+    configs = [
+        {
+            "test_name": "test_model",
+            "server_params": {"model": "some/model"},
+            "benchmark_params": [
+                {
+                    "task": "voice_clone",
+                    "enabled": True,
+                    "dataset_name": "seed-tts",
+                    "backend": "openai-audio-speech",
+                    "endpoint": "/v1/audio/speech",
+                    "num_prompts": [10],
+                    "max_concurrency": [1],
+                    "percentile-metrics": "audio_rtf",
+                    "baseline": {},
+                },
+                {
+                    "task": "voice_design",
+                    "enabled": False,
+                    "dataset_name": "seed-tts-design",
+                    "backend": "openai-audio-speech",
+                    "endpoint": "/v1/audio/speech",
+                    "num_prompts": [5],
+                    "max_concurrency": [1],
+                    "percentile-metrics": "audio_rtf",
+                    "baseline": {},
+                },
+            ],
+        }
+    ]
+    mapping = create_test_parameter_mapping(configs)
+    params = mapping["test_model"]["benchmark_params"]
+    # Only the enabled=True entry should appear
+    assert len(params) == 1
+    assert params[0].get("task") == "voice_clone"

tests/dfx/perf/tests/test_tts.json

@@ -1,34 +1,155 @@
 [
-    {
-        "test_name": "test_qwen3_tts",
-        "server_params": {
-            "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
-        },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "openai-audio-speech",
-                "endpoint": "/v1/audio/speech",
-                "num_prompts": [
-                    10,
-                    40
-                ],
-                "max_concurrency": [
-                    1,
-                    4
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "extra_body": {
-                    "voice": "Vivian",
-                    "language": "English"
-                },
-                "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_audio_ttfp_ms": [6000, 6000],
-                    "mean_audio_rtf": [0.3, 0.3]
-                }
-            }
-        ]
-    }
+  {
+    "test_name": "test_qwen3_tts_base",
+    "server_params": {
+      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+    },
+    "benchmark_params": [
+      {
+        "task": "voice_clone",
+        "eval_phase": "latency",
+        "enabled": false,
+        "dataset_name": "seed-tts",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "num_prompts": [20],
+        "max_concurrency": [1],
+        "seed_tts_locale": "en",
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [350],
+          "median_audio_rtf": [0.25]
+        }
+      },
+      {
+        "task": "voice_clone",
+        "eval_phase": "throughput",
+        "enabled": false,
+        "dataset_name": "seed-tts",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "num_prompts": [80],
+        "max_concurrency": [8],
+        "seed_tts_locale": "en",
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [3500],
+          "median_audio_rtf": [0.75],
+          "audio_throughput": [10.0]
+        }
+      },
+      {
+        "task": "voice_clone",
+        "eval_phase": "quality",
+        "enabled": false,
+        "dataset_name": "seed-tts",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "num_prompts": [200],
+        "max_concurrency": [4],
+        "seed_tts_locale": "en",
+        "seed_tts_wer_eval": true,
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_audio_rtf": [0.45]
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen3_tts_customvoice",
+    "server_params": {
+      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+    },
+    "benchmark_params": [
+      {
+        "task": "default_voice",
+        "eval_phase": "latency",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [20],
+        "max_concurrency": [1],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [150],
+          "median_audio_rtf": [0.15]
+        }
+      },
+      {
+        "task": "default_voice",
+        "eval_phase": "throughput",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [80],
+        "max_concurrency": [8],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [1500],
+          "median_audio_rtf": [0.30],
+          "audio_throughput": [30.0]
+        }
+      },
+      {
+        "task": "default_voice",
+        "eval_phase": "quality",
+        "enabled": false,
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [200],
+        "max_concurrency": [4],
+        "seed_tts_locale": "en",
+        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "seed_tts_wer_eval": true,
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_audio_rtf": [0.35]
+        }
+      },
+      {
+        "task": "voice_design",
+        "eval_phase": "latency",
+        "dataset_name": "seed-tts-design",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_design",
+        "num_prompts": [20],
+        "max_concurrency": [1],
+        "seed_tts_locale": "en",
+        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [150],
+          "median_audio_rtf": [0.15]
+        }
+      },
+      {
+        "task": "voice_design",
+        "eval_phase": "throughput",
+        "dataset_name": "seed-tts-design",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_design",
+        "num_prompts": [80],
+        "max_concurrency": [8],
+        "seed_tts_locale": "en",
+        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [1500],
+          "median_audio_rtf": [0.35],
+          "audio_throughput": [25.0]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/reliability/README.md

@@ -0,0 +1,14 @@
+# L5(b) Reliability (`tests/dfx/reliability`)
+
+This directory contains reliability fault-injection tests for key models.
+The detailed RFC document is maintained outside this repository (local/internal only).
+
+- Fault helpers: `conftest.py`, `helpers.py`
+- Qwen3 tests: `test_reliability_qwen3_omni.py`
+- Wan2.2 tests: `test_reliability_wan22.py`
+
+```bash
+pytest --collect-only tests/dfx/reliability
+pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
+```

tests/dfx/reliability/conftest.py

@@ -0,0 +1,29 @@
+"""Pytest fixtures for reliability tests."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.dfx.reliability.helpers import FaultInjector
+
+
+@pytest.fixture
+def fault_injector(request: pytest.FixtureRequest) -> FaultInjector:
+    """Indirect only: ``request.param`` must be a ``FaultInjector`` callable."""
+    return request.param
+
+
+@pytest.fixture
+def omni_server_after_fault(omni_server: Any, fault_injector: FaultInjector):
+    """After ``omni_server`` is up, run ``fault_injector(omni_server)``, then yield the server."""
+    fault_injector(omni_server)
+    yield omni_server
+
+
+@pytest.fixture
+def omni_server_after_fault_function(omni_server_function: Any, fault_injector: FaultInjector):
+    """Inject fault after function-scoped server startup, then yield server."""
+    fault_injector(omni_server_function)
+    yield omni_server_function

tests/dfx/reliability/helpers.py

@@ -0,0 +1,567 @@
+"""Shared reliability fault-injection helpers.
+
+This module keeps fault injection callable from tests directly:
+- GPU OOM (CUDA sidecar memory hog)
+- process kill by pattern and signal
+- post-ready hooks via ``fault_injector`` / ``omni_server_after_fault`` fixtures
+"""
+
+from __future__ import annotations
+
+import http.client
+import json
+import logging
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import psutil
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class OomHandle:
+    """Handle for a started CUDA memory hog subprocess."""
+
+    proc: subprocess.Popen | None
+    device: int
+    target_mem_ratio: float
+    start_ts: float
+
+
+def post_chat_completions_raw(
+    host: str,
+    port: int,
+    body: bytes | str,
+    *,
+    content_type: str = "application/json",
+    timeout_sec: int = 120,
+) -> tuple[int, bytes]:
+    """POST /v1/chat/completions with raw bytes; returns (status, response_body)."""
+    conn = http.client.HTTPConnection(host, port, timeout=timeout_sec)
+    try:
+        headers = {"Content-Type": content_type}
+        payload = body.encode("utf-8") if isinstance(body, str) else body
+        conn.request("POST", "/v1/chat/completions", body=payload, headers=headers)
+        resp = conn.getresponse()
+        data = resp.read()
+        return resp.status, data
+    finally:
+        conn.close()
+
+
+def get_health_raw(host: str, port: int, *, timeout_sec: int = 20) -> tuple[int, bytes]:
+    """GET /health with stdlib HTTP client; returns (status, body)."""
+    conn = http.client.HTTPConnection(host, port, timeout=timeout_sec)
+    try:
+        conn.request("GET", "/health")
+        resp = conn.getresponse()
+        return resp.status, resp.read()
+    finally:
+        conn.close()
+
+
+def post_json_raw_http_client(
+    host: str,
+    port: int,
+    path: str,
+    payload: dict[str, Any],
+    *,
+    timeout_sec: int = 30,
+) -> tuple[int, bytes]:
+    """POST JSON to one endpoint with stdlib HTTP client; returns (status, body)."""
+    conn = http.client.HTTPConnection(host, port, timeout=timeout_sec)
+    try:
+        body = json.dumps(payload).encode("utf-8")
+        conn.request("POST", path, body=body, headers={"Content-Type": "application/json"})
+        resp = conn.getresponse()
+        return resp.status, resp.read()
+    finally:
+        conn.close()
+
+
+def post_json_raw(
+    host: str,
+    port: int,
+    path: str,
+    payload: dict[str, Any],
+    *,
+    timeout_sec: int = 30,
+) -> tuple[int, bytes]:
+    """POST JSON to one endpoint; returns (status, body)."""
+    return (
+        post_chat_completions_raw(
+            host,
+            port,
+            json.dumps(payload),
+            content_type="application/json",
+            timeout_sec=timeout_sec,
+        )
+        if path == "/v1/chat/completions"
+        else post_json_raw_http_client(
+            host,
+            port,
+            path,
+            payload,
+            timeout_sec=timeout_sec,
+        )
+    )
+
+
+def extract_openai_error_contract_from_bytes(response_body: bytes) -> dict[str, Any] | None:
+    """Best-effort parse OpenAI-style error object from raw response bytes."""
+    try:
+        payload = json.loads(response_body.decode("utf-8", errors="replace"))
+    except Exception:  # noqa: BLE001
+        return None
+    return extract_openai_error_contract_from_payload(payload)
+
+
+def extract_openai_error_contract_from_payload(payload: Any) -> dict[str, Any] | None:
+    """Best-effort parse OpenAI-style error object from decoded JSON payload."""
+    if not isinstance(payload, dict):
+        return None
+    error_obj = payload.get("error")
+    if not isinstance(error_obj, dict):
+        return None
+    if not isinstance(error_obj.get("message"), str):
+        return None
+    return error_obj
+
+
+def _build_sidecar_cmd(device: int, target_mem_ratio: float, hold_seconds: int, strict: bool) -> list[str]:
+    sidecar = r"""
+import sys
+import time
+import torch
+
+device = int(sys.argv[1])
+target_ratio = float(sys.argv[2])
+hold_seconds = int(sys.argv[3])
+strict = sys.argv[4] == "1"
+
+torch.cuda.init()
+torch.cuda.set_device(device)
+props = torch.cuda.get_device_properties(device)
+free_before, total_bytes = torch.cuda.mem_get_info(device)
+target_bytes = int(free_before * target_ratio)
+chunk_bytes = 256 * 1024 * 1024
+chunks = []
+allocated = 0
+
+while allocated < target_bytes:
+    req_bytes = min(chunk_bytes, target_bytes - allocated)
+    req_elems = max(1, req_bytes // 2)  # float16 -> 2 bytes
+    try:
+        chunk = torch.empty((req_elems,), dtype=torch.float16, device=f"cuda:{device}")
+        chunks.append(chunk)
+        allocated += chunk.numel() * 2
+    except RuntimeError:
+        break
+
+# In strict mode, keep filling with smaller chunks until allocator rejects.
+# This minimizes residual free memory and makes fault-path assertions steadier.
+if strict:
+    tail_chunk_bytes = [64 * 1024 * 1024, 16 * 1024 * 1024, 4 * 1024 * 1024, 1 * 1024 * 1024]
+    for tail_bytes in tail_chunk_bytes:
+        while True:
+            req_elems = max(1, tail_bytes // 2)
+            try:
+                chunk = torch.empty((req_elems,), dtype=torch.float16, device=f"cuda:{device}")
+                chunks.append(chunk)
+                allocated += chunk.numel() * 2
+            except RuntimeError:
+                break
+
+achieved_ratio = allocated / max(1, props.total_memory)
+achieved_free_ratio = allocated / max(1, int(free_before))
+free_after, _ = torch.cuda.mem_get_info(device)
+if strict and allocated < target_bytes:
+    print(
+        "ERROR:"
+        f"achieved_free_ratio={achieved_free_ratio:.4f};"
+        f"achieved_total_ratio={achieved_ratio:.4f};"
+        f"free_before={int(free_before)};"
+        f"free_after={int(free_after)};"
+        f"target_bytes={target_bytes};"
+        f"allocated={allocated}",
+        flush=True,
+    )
+    sys.exit(2)
+
+print(
+    "READY:"
+    f"achieved_free_ratio={achieved_free_ratio:.4f};"
+    f"achieved_total_ratio={achieved_ratio:.4f};"
+    f"free_before={int(free_before)};"
+    f"free_after={int(free_after)};"
+    f"target_bytes={target_bytes};"
+    f"allocated={allocated}",
+    flush=True,
+)
+if hold_seconds <= 0:
+    while True:
+        time.sleep(3600)
+time.sleep(hold_seconds)
+print("DONE", flush=True)
+"""
+    return [
+        sys.executable,
+        "-u",
+        "-c",
+        sidecar,
+        str(device),
+        str(target_mem_ratio),
+        str(hold_seconds),
+        "1" if strict else "0",
+    ]
+
+
+def start_gpu_oom_hog(
+    *,
+    device: int = 0,
+    target_mem_ratio: float = 0.95,
+    hold_seconds: int = 60,
+    startup_timeout_sec: int = 20,
+    strict: bool = True,
+    poll_interval_sec: float = 0.2,
+) -> OomHandle:
+    """Start a CUDA sidecar process that occupies GPU memory to trigger OOM.
+
+    Note:
+        ``target_mem_ratio`` is evaluated against free memory at injection start
+        (not total GPU memory), i.e. success gate is ``allocated / free_before``.
+        ``hold_seconds <= 0`` means keeping OOM pressure until the sidecar is
+        explicitly stopped via ``stop_gpu_oom_hog(s)``.
+    """
+    if os.name == "nt":
+        raise RuntimeError("CUDA OOM sidecar is intended for Linux CI/runtime.")
+    if not (0.0 <= target_mem_ratio < 1.0):
+        raise ValueError("target_mem_ratio should be in [0.0, 1.0).")
+
+    # Explicit opt-out for debugging: keep API shape stable while disabling injection.
+    if target_mem_ratio == 0.0:
+        print(f"[oom-sidecar][gpu={device}] DISABLED: target_mem_ratio=0.0 (no OOM injection)", flush=True)
+        return OomHandle(
+            proc=None,
+            device=device,
+            target_mem_ratio=target_mem_ratio,
+            start_ts=time.time(),
+        )
+
+    cmd = _build_sidecar_cmd(device, target_mem_ratio, hold_seconds, strict)
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+    )
+    assert proc.stdout is not None
+
+    deadline = time.time() + startup_timeout_sec
+    logs: list[str] = []
+    while time.time() < deadline:
+        ready, _, _ = select.select([proc.stdout], [], [], poll_interval_sec)
+        if ready:
+            line = proc.stdout.readline().strip()
+            if line:
+                logs.append(line)
+                print(f"[oom-sidecar][gpu={device}] {line}", flush=True)
+                if line.startswith("READY:"):
+                    return OomHandle(
+                        proc=proc,
+                        device=device,
+                        target_mem_ratio=target_mem_ratio,
+                        start_ts=time.time(),
+                    )
+                if line.startswith("ERROR:"):
+                    proc.terminate()
+                    raise RuntimeError(f"OOM sidecar failed to reach target: {line}")
+        if proc.poll() is not None:
+            break
+
+    proc.terminate()
+    if logs:
+        print(f"[oom-sidecar][gpu={device}] startup logs: {' | '.join(logs)}", flush=True)
+    raise TimeoutError(f"OOM sidecar startup timeout. logs={logs}")
+
+
+def stop_gpu_oom_hog(handle: OomHandle, *, timeout_sec: int = 5) -> None:
+    """Stop and cleanup CUDA OOM sidecar."""
+    proc = handle.proc
+    if proc is None:
+        return
+    if proc.poll() is not None:
+        return
+    proc.terminate()
+    try:
+        proc.wait(timeout=timeout_sec)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait(timeout=timeout_sec)
+
+
+def inject_gpu_oom(
+    *,
+    device: int | str | list[int] = 0,
+    target_mem_ratio: float = 0.95,
+    hold_seconds: int = 60,
+    startup_timeout_sec: int = 20,
+    strict: bool = True,
+) -> OomHandle | list[OomHandle]:
+    """Convenience wrapper to start CUDA OOM sidecar(s).
+
+    Args:
+        device: One device id (``0``), comma-separated string (``"0,1,2"``),
+            or a list of device ids (``[0, 1, 2]``).
+        hold_seconds: OOM hold time in seconds; ``<=0`` keeps pressure until
+            ``stop_gpu_oom_hogs`` is called.
+    """
+    if isinstance(device, int):
+        devices = [device]
+    elif isinstance(device, str):
+        devices = [int(x.strip()) for x in device.split(",") if x.strip()]
+    else:
+        devices = [int(x) for x in device]
+    if not devices:
+        raise ValueError("device must not be empty.")
+
+    handles = [
+        start_gpu_oom_hog(
+            device=dev,
+            target_mem_ratio=target_mem_ratio,
+            hold_seconds=hold_seconds,
+            startup_timeout_sec=startup_timeout_sec,
+            strict=strict,
+        )
+        for dev in devices
+    ]
+    if len(handles) == 1:
+        return handles[0]
+    return handles
+
+
+def stop_gpu_oom_hogs(handles: OomHandle | list[OomHandle], *, timeout_sec: int = 5) -> None:
+    """Stop one or multiple OOM sidecars."""
+    if isinstance(handles, OomHandle):
+        stop_gpu_oom_hog(handles, timeout_sec=timeout_sec)
+        return
+    for handle in handles:
+        stop_gpu_oom_hog(handle, timeout_sec=timeout_sec)
+
+
+def _runtime_teardown_ssh_target() -> str:
+    target = os.getenv("RUNTIME_TEARDOWN_SSH_TARGET", "").strip()
+    # Default to root@127.0.0.1 for same-host SSH control path.
+    return target or "root@127.0.0.1"
+
+
+def _runtime_teardown_ssh_cmd(remote_cmd: str, *, step: str | None = None) -> subprocess.CompletedProcess[str]:
+    ssh_target = _runtime_teardown_ssh_target()
+    default_reuse_opts = "-o ControlMaster=auto -o ControlPersist=10m -o ControlPath=/tmp/vllm-rt-ssh-%r@%h:%p"
+    raw_opts = os.getenv("RUNTIME_TEARDOWN_SSH_OPTS", "").strip()
+    ssh_opts = shlex.split(raw_opts or default_reuse_opts)
+    timeout_sec = int(os.getenv("RUNTIME_TEARDOWN_SSH_TIMEOUT_SEC", "600"))
+    step_prefix = f"[runtime-teardown][ssh]{f'[{step}]' if step else ''}"
+    print(f"{step_prefix} target={ssh_target} running remote command...", flush=True)
+    # IMPORTANT: SSH joins remote argv into one shell command string. If we pass
+    # ["bash", "-c", remote_cmd] as separate argv items, remote shell parsing can
+    # make `-c` consume only the first word (e.g. "docker"), causing docker help.
+    # Wrap the whole command as one quoted string for remote bash -c.
+    remote_invocation = f"bash --noprofile --norc -c {shlex.quote(remote_cmd)}"
+    try:
+        out = subprocess.run(
+            ["ssh", *ssh_opts, ssh_target, remote_invocation],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=timeout_sec,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(
+            f"{step_prefix} timed out after {timeout_sec}s. Increase RUNTIME_TEARDOWN_SSH_TIMEOUT_SEC if needed."
+        ) from exc
+    print(f"{step_prefix} exit_code={out.returncode}", flush=True)
+    return out
+
+
+def list_remote_process_pids_by_pattern(pattern: str) -> list[int]:
+    """Return matched PIDs from remote host ``pgrep -f <pattern>`` via SSH."""
+    cmd = f"pgrep -f {shlex.quote(pattern)} || true"
+    out = _runtime_teardown_ssh_cmd(cmd, step="pgrep")
+    if out.returncode not in (0, 1):
+        raise RuntimeError(f"remote pgrep failed for pattern={pattern!r}: {out.stderr.strip()}")
+    return [int(item) for item in out.stdout.split() if item.strip().isdigit()]
+
+
+def inject_process_kill(
+    *,
+    grep_pattern: str,
+    signal_name: str = "SIGTERM",
+    limit: int | None = None,
+    allow_zero_match: bool = False,
+    execute_kill: bool = True,
+) -> list[int]:
+    """Kill processes matching pattern with selected signal."""
+    if os.name == "nt":
+        raise RuntimeError("process-kill helper currently supports POSIX platforms only.")
+    if not grep_pattern.strip():
+        raise ValueError("grep_pattern must not be empty.")
+
+    sig = getattr(signal, signal_name, None)
+    if sig is None:
+        raise ValueError(f"Unsupported signal_name: {signal_name}")
+
+    out = subprocess.run(
+        ["pgrep", "-f", grep_pattern],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    pids = [int(item) for item in out.stdout.split() if item.strip().isdigit()]
+    if limit is not None:
+        pids = pids[:limit]
+
+    if not pids and not allow_zero_match:
+        raise RuntimeError(f"No process matched pattern: {grep_pattern}")
+
+    if execute_kill:
+        for pid in pids:
+            os.kill(pid, sig)
+    return pids
+
+
+def _safe_proc_info(pid: int) -> tuple[str, str]:
+    """Best-effort process name/cmdline lookup for debug logging."""
+    try:
+        proc = psutil.Process(pid)
+        name = proc.name()
+        cmdline = " ".join(proc.cmdline()) or "<empty-cmdline>"
+        return name, cmdline
+    except Exception:  # noqa: BLE001
+        return "<unknown>", "<unavailable>"
+
+
+def _list_server_process_tree(server: Any) -> list[int]:
+    """Return [root, descendants...] PIDs for the current test server instance."""
+    root_proc = getattr(server, "proc", None)
+    if root_proc is None or getattr(root_proc, "pid", None) is None:
+        return []
+
+    root_pid = int(root_proc.pid)
+    try:
+        root = psutil.Process(root_pid)
+    except Exception:  # noqa: BLE001
+        return [root_pid]
+
+    descendants = [child.pid for child in root.children(recursive=True)]
+    return [root_pid, *descendants]
+
+
+def _log_server_process_tree(server: Any) -> None:
+    """Print server process tree for debugging fault injection targets."""
+    pids = _list_server_process_tree(server)
+    if not pids:
+        logger.warning("[reliability][process-kill] current server has no visible process tree")
+        return
+    for pid in pids:
+        name, cmdline = _safe_proc_info(pid)
+        print(
+            f"[reliability][process-kill] current_server_proc pid={pid} name={name} cmdline={cmdline}",
+            flush=True,
+        )
+
+
+FaultInjector = Callable[[Any], None]
+"""Callable invoked with the live ``OmniServer`` after it is ready (see ``omni_server_after_fault``)."""
+
+
+def make_process_kill_fault_injector(
+    *,
+    grep_patterns: str | Sequence[str],
+    signal_name: str = "SIGKILL",
+    limit: int = 1,
+    post_kill_wait_seconds: float = 0.0,
+) -> FaultInjector:
+    """Build a post-ready injector that kills processes matched by ``pgrep -f``.
+
+    Tries each pattern in order until at least one PID is killed. If none match,
+    the returned callable issues ``pytest.skip`` (same behavior as the previous
+    inline reliability test).
+
+    Args:
+        grep_patterns: One pattern or an ordered list of patterns.
+        signal_name: Passed to :func:`inject_process_kill` (e.g. ``SIGKILL``).
+        limit: Maximum PIDs to kill per pattern (default ``1``).
+        post_kill_wait_seconds: Optional wait time after kill before test request starts.
+    """
+    patterns: tuple[str, ...] = (grep_patterns,) if isinstance(grep_patterns, str) else tuple(grep_patterns)
+
+    def _inject(server: Any) -> None:
+        _log_server_process_tree(server)
+        server_tree = set(_list_server_process_tree(server))
+        if not server_tree:
+            logger.warning(
+                "[reliability][process-kill] no server process tree found; fallback to global pgrep matching"
+            )
+        for pattern in patterns:
+            print(
+                f"[reliability][process-kill] trying pattern={pattern} signal={signal_name} limit={limit}",
+                flush=True,
+            )
+            pids = inject_process_kill(
+                grep_pattern=pattern,
+                signal_name=signal_name,
+                limit=limit,
+                allow_zero_match=True,
+                execute_kill=False,
+            )
+            filtered = [pid for pid in pids if not server_tree or pid in server_tree]
+            if pids and not filtered:
+                logger.warning(
+                    "[reliability][process-kill] pattern=%s matched non-server pids=%s, skip them",
+                    pattern,
+                    pids,
+                )
+                continue
+            if filtered:
+                sig = getattr(signal, signal_name, None)
+                if sig is None:
+                    raise ValueError(f"Unsupported signal_name: {signal_name}")
+                for pid in filtered:
+                    name, cmdline = _safe_proc_info(pid)
+                    print(
+                        f"[reliability][process-kill] killing pid={pid} name={name} signal={signal_name} cmdline={cmdline}",
+                        flush=True,
+                    )
+                    os.kill(pid, sig)
+                print(
+                    f"[reliability][process-kill] matched pattern={pattern} killed_pids={filtered} killed_count={len(filtered)}",
+                    flush=True,
+                )
+                if post_kill_wait_seconds > 0:
+                    print(
+                        f"[reliability][process-kill] waiting {post_kill_wait_seconds:.2f}s after kill",
+                        flush=True,
+                    )
+                    time.sleep(post_kill_wait_seconds)
+                return
+        logger.warning(
+            "[reliability][process-kill] no process matched patterns=%s signal=%s limit=%s",
+            patterns,
+            signal_name,
+            limit,
+        )
+        pytest.skip("no matching runtime process found for kill injection")
+
+    return _inject

tests/dfx/reliability/test_reliability_qwen3_omni.py

@@ -0,0 +1,682 @@
+"""
+Qwen3-Omni reliability integration tests.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import errno
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, Protocol
+
+import pytest
+import torch
+
+from tests.dfx.conftest import (
+    assert_fault_exception,
+    create_reliability_omni_server_params,
+    resolve_oom_device_spec,
+)
+from tests.dfx.reliability.helpers import (
+    extract_openai_error_contract_from_bytes,
+    get_health_raw,
+    inject_gpu_oom,
+    make_process_kill_fault_injector,
+    post_chat_completions_raw,
+    post_json_raw,
+    stop_gpu_oom_hogs,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
+from tests.helpers.runtime import dummy_messages_from_mix_data
+
+RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
+    {
+        "test_name": "qwen3_omni_reliability_async_chunk",
+        "server_params": {
+            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+            "stage_config_name": "qwen3_omni_moe.yaml",
+            "server_args": ["--async-chunk"],
+        },
+    },
+    {
+        "test_name": "qwen3_omni_reliability_default",
+        "server_params": {
+            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+            "stage_config_name": "qwen3_omni_moe.yaml",
+            "server_args": ["--no-async-chunk"],
+        },
+    },
+]
+
+DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy"
+
+
+def _default_oom_device_spec() -> str:
+    """Use currently visible CUDA ordinals to avoid invalid device index in sidecar."""
+    count = torch.cuda.device_count()
+    if count <= 0:
+        return "0"
+    return ",".join(str(i) for i in range(count))
+
+
+OOM_INJECTION_CONFIG = {
+    "device": _default_oom_device_spec(),
+    "target_mem_ratio": 0.92,
+    "hold_seconds": 0,
+    "startup_timeout_sec": 20,
+    "strict": False,
+}
+# Post-fault recovery probe: keep the API process alive on the same GPU.
+# ``strict=True`` sidecars keep allocating until CUDA refuses, which often
+# evicts/kills the server; here we only take a fraction of *initial* free
+# memory and stop, then rely on a heavy multimodal forward to hit OOM.
+OOM_RECOVER_INJECTION_CONFIG = {
+    "device": _default_oom_device_spec(),
+    # ``strict=False`` stops at this fraction of *initial* free memory per GPU (see helpers sidecar).
+    # On multi-GPU hosts a mild ratio plus a small e2e-style mix can still leave several GB free
+    # on a data-parallel device and the request succeeds; 0.92+ pairs better with the heavy
+    # probe payload below. Raise toward 0.94 only if fault phase still never fails; avoid strict=True
+    # here so the API process is not squeezed to zero slack.
+    "target_mem_ratio": 0.8,
+    "hold_seconds": 0,
+    "startup_timeout_sec": 20,
+    "strict": False,
+}
+FAULT_ERROR_KEYWORDS = (
+    "the request failed",
+    "oom",
+    "out of memory",
+    "cuda",
+    "orchestrator",
+    "timeout",
+    "connection",
+    "500",
+    "503",
+)
+RUNTIME_WORKER_PATTERN = "VLLM::Worker"
+
+
+class _HasServeArgs(Protocol):
+    model: str
+    serve_args: list[str]
+
+
+def _get_system_prompt() -> dict:
+    return {
+        "role": "system",
+        "content": [
+            {
+                "type": "text",
+                "text": (
+                    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+                    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
+                ),
+            }
+        ],
+    }
+
+
+def _get_mix_prompt() -> str:
+    return "What is recited in the audio? What is in this image? Describe the video briefly."
+
+
+def _mix_chat_completions_probe_payload(omni_server: _HasServeArgs) -> dict[str, Any]:
+    """Heavy multimodal ``/v1/chat/completions`` body for OOM recover tests.
+
+    Uses the same synthetic sizes as ``test_reliability_fault_gpu_oom_chat_large_payload_failure``
+    (large vision tensors + long text), not the light expansion mix (224² / 5s audio): under
+    ``strict=False`` sidecars, a small mix often still completes on a GPU that retains multi-GB
+    free after the hog stops at ``target_mem_ratio``.
+    """
+    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(1280, 720, 161)['base64']}"
+    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(20, 1)['base64']}"
+    messages = dummy_messages_from_mix_data(
+        system_prompt=_get_system_prompt(),
+        video_data_url=video_data_url,
+        image_data_url=image_data_url,
+        audio_data_url=audio_data_url,
+        content_text="What is recited in the audio? What is in this image? What is in this video? " * 200,
+    )
+    return {
+        "model": omni_server.model,
+        "messages": messages,
+        "stream": True,
+        "modalities": ["text", "audio"],
+    }
+
+
+def _fault_keywords_match_response(*, status: int, body: bytes) -> bool:
+    """True when HTTP status or body text looks like an expected fault / OOM class outcome."""
+    if status >= 400:
+        return True
+    text = body.decode("utf-8", errors="replace").lower()
+    # Avoid matching bare ``"500"`` / ``"503"`` in successful streamed bodies (base64/SSE noise).
+    body_fault_hints = (
+        "the request failed",
+        "oom",
+        "out of memory",
+        "cuda",
+        "orchestrator",
+        "timeout",
+        "connection refused",
+        "connection reset",
+    )
+    return any(key in text for key in body_fault_hints)
+
+
+def _stage_config_path_from_omni_server(omni_server: _HasServeArgs) -> str | None:
+    args: list[str] = omni_server.serve_args
+    for i, arg in enumerate(args):
+        if arg == "--stage-configs-path" and i + 1 < len(args):
+            return args[i + 1]
+        if arg.startswith("--stage-configs-path="):
+            return arg.split("=", 1)[1]
+    return None
+
+
+def _looks_like_server_unreachable(exc: BaseException) -> bool:
+    """True when /health cannot be reached because nothing is listening (process exited)."""
+    if isinstance(exc, (ConnectionRefusedError, BrokenPipeError, ConnectionResetError)):
+        return True
+    errno_val = getattr(exc, "errno", None)
+    if isinstance(exc, OSError) and errno_val is not None:
+        return errno_val in (
+            errno.ECONNREFUSED,
+            errno.ECONNRESET,
+            errno.EPIPE,
+        )
+    msg = str(exc).lower()
+    return "connection refused" in msg or "actively refused" in msg
+
+
+QWEN_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR)
+
+
+@pytest.mark.slow
+@hardware_test(res={"cuda": "H100"}, num_cards=2)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_error_contract_consistent_chat_speech(
+    omni_server_function,
+) -> None:
+    """Black-box: text chat vs omni audio output should expose a consistent error contract under OOM.
+
+    Speech-style output is requested via ``/v1/chat/completions`` with ``modalities`` that include ``audio``.
+    """
+    device_spec = resolve_oom_device_spec(
+        OOM_INJECTION_CONFIG,
+        _stage_config_path_from_omni_server(omni_server_function),
+    )
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_INJECTION_CONFIG["strict"],
+    )
+    host = omni_server_function.host
+    port = omni_server_function.port
+    mix_base = _mix_chat_completions_probe_payload(omni_server_function)
+    oom_contract_timeout_sec = 120
+    try:
+        chat_status, chat_body = post_json_raw(
+            host,
+            port,
+            "/v1/chat/completions",
+            {
+                "model": mix_base["model"],
+                "messages": mix_base["messages"],
+                "stream": False,
+                "modalities": ["text"],
+            },
+            timeout_sec=oom_contract_timeout_sec,
+        )
+        speech_status, speech_body = post_json_raw(
+            host,
+            port,
+            "/v1/chat/completions",
+            {
+                "model": mix_base["model"],
+                "messages": mix_base["messages"],
+                "stream": False,
+                "modalities": ["text", "audio"],
+            },
+            timeout_sec=oom_contract_timeout_sec,
+        )
+    finally:
+        stop_gpu_oom_hogs(handle)
+
+    # Under OOM pressure both chat and speech should surface runtime-class (5xx)
+    # failures instead of request-validation-class (4xx) errors.
+    chat_error = extract_openai_error_contract_from_bytes(chat_body)
+    speech_error = extract_openai_error_contract_from_bytes(speech_body)
+    print(chat_status, chat_error, speech_status, speech_error)
+
+    assert chat_status >= 500, f"expected chat error under OOM, got status={chat_status}"
+    assert speech_status >= 500, f"expected speech runtime error under OOM, got status={speech_status}"
+
+    assert chat_error is not None, f"chat error payload not OpenAI-compatible: {chat_body[:300]!r}"
+    assert speech_error is not None, f"speech error payload not OpenAI-compatible: {speech_body[:300]!r}"
+    assert "code" in chat_error, f"chat error lacks code field: {chat_error!r}"
+    assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}"
+
+
+@pytest.mark.slow
+@hardware_test(res={"cuda": "H100"}, num_cards=2)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_chat_large_payload_failure(omni_server_function, openai_client_function) -> None:
+    device_spec = resolve_oom_device_spec(
+        OOM_INJECTION_CONFIG,
+        _stage_config_path_from_omni_server(omni_server_function),
+    )
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_INJECTION_CONFIG["strict"],
+    )
+    try:
+        video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(1280, 720, 161)['base64']}"
+        image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+        audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(20, 1)['base64']}"
+        messages = dummy_messages_from_mix_data(
+            system_prompt=_get_system_prompt(),
+            video_data_url=video_data_url,
+            image_data_url=image_data_url,
+            audio_data_url=audio_data_url,
+            content_text=f"{_get_mix_prompt()} " * 200,
+        )
+        request_config = {
+            "model": omni_server_function.model,
+            "messages": messages,
+            "stream": True,
+            "key_words": {"audio": ["test"]},
+        }
+        try:
+            openai_client_function.send_omni_request(request_config, request_num=1)
+        except Exception as exc:
+            assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+        else:
+            pytest.fail("expected large chat payload request failure during GPU OOM injection")
+    finally:
+        stop_gpu_oom_hogs(handle)
+
+
+@pytest.mark.slow
+@hardware_test(res={"cuda": "H100"}, num_cards=2)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_concurrent_pressure_failure(omni_server_function, openai_client_function) -> None:
+    device_spec = resolve_oom_device_spec(
+        OOM_INJECTION_CONFIG,
+        _stage_config_path_from_omni_server(omni_server_function),
+    )
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_INJECTION_CONFIG["strict"],
+    )
+    try:
+        video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(1280, 720, 161)['base64']}"
+        image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+        audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(20, 1)['base64']}"
+        messages = dummy_messages_from_mix_data(
+            system_prompt=_get_system_prompt(),
+            video_data_url=video_data_url,
+            image_data_url=image_data_url,
+            audio_data_url=audio_data_url,
+            content_text=f"{_get_mix_prompt()} " * 200,
+        )
+        request_config = {
+            "model": omni_server_function.model,
+            "messages": messages,
+            "stream": True,
+            "modalities": ["text", "audio"],
+            "key_words": {"audio": ["test"]},
+        }
+        try:
+            openai_client_function.send_omni_request(request_config, request_num=4)
+        except Exception as exc:
+            assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+        else:
+            pytest.fail("expected concurrent request failure under OOM injection")
+    finally:
+        stop_gpu_oom_hogs(handle)
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+    "fault_injector",
+    [
+        pytest.param(
+            make_process_kill_fault_injector(
+                grep_patterns="VLLM::Worker",
+                signal_name="SIGKILL",
+                limit=1,
+                post_kill_wait_seconds=2.0,
+            ),
+            id="runtime_process_chain",
+        ),
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_request_failure(
+    omni_server_after_fault_function, openai_client_function
+) -> None:
+    messages = dummy_messages_from_mix_data(
+        system_prompt=_get_system_prompt(),
+        content_text="What is the capital of China? Answer in 20 words.",
+    )
+    request_config = {
+        "model": omni_server_after_fault_function.model,
+        "messages": messages,
+        "stream": False,
+        "modalities": ["text"],
+        "key_words": {"text": ["beijing"]},
+    }
+    try:
+        openai_client_function.send_omni_request(request_config, request_num=1)
+    except Exception as exc:
+        assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+    else:
+        pytest.fail("expected request failure after process-kill injection")
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+    "fault_injector",
+    [
+        pytest.param(
+            make_process_kill_fault_injector(
+                grep_patterns="VLLM::Worker",
+                signal_name="SIGKILL",
+                limit=1,
+                post_kill_wait_seconds=2.0,
+            ),
+            id="runtime_process_chain",
+        ),
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
+    omni_server_after_fault_function,
+) -> None:
+    """Black-box: after worker SIGKILL, /health→503, chat fails fast, concurrent chat does not hang."""
+    host = omni_server_after_fault_function.host
+    port = omni_server_after_fault_function.port
+    model = omni_server_after_fault_function.model
+
+    deadline = time.monotonic() + 20.0
+    last_observation = ""
+    saw_503 = False
+    health_final_status: int | None = None
+    health_final_body = b""
+    while time.monotonic() < deadline:
+        try:
+            status, body = get_health_raw(host, port, timeout_sec=5)
+            last_observation = f"http={status}, body={body[:200]!r}"
+            health_final_status, health_final_body = status, body
+            if status == 503:
+                saw_503 = True
+                break
+        except Exception as exc:  # noqa: BLE001
+            last_observation = f"exception={exc!r}"
+        time.sleep(0.5)
+    assert saw_503, (
+        f"[process_kill health] expected /health to become 503 after fault injection, got {last_observation}"
+    )
+
+    payload = {
+        "model": model,
+        "messages": [{"role": "user", "content": "Say hello in one short sentence."}],
+        "stream": False,
+        "modalities": ["text"],
+    }
+    ff_status: int | None = None
+    ff_body = b""
+    ff_exc: BaseException | None = None
+    start = time.monotonic()
+    try:
+        ff_status, ff_body = post_json_raw(host, port, "/v1/chat/completions", payload, timeout_sec=20)
+        elapsed = time.monotonic() - start
+        assert elapsed < 15, f"[process_kill fast_fail] request did not fail fast after fault: {elapsed:.2f}s"
+        assert ff_status >= 500, (
+            f"[process_kill fast_fail] expected server-side failure after fault, "
+            f"got status={ff_status}, body={ff_body[:200]!r}"
+        )
+    except Exception as exc:
+        ff_exc = exc
+        elapsed = time.monotonic() - start
+        assert elapsed < 15, f"[process_kill fast_fail] request exception was too slow after fault: {elapsed:.2f}s"
+
+    payload_json = json.dumps(
+        {
+            "model": model,
+            "messages": [{"role": "user", "content": "What is the capital of China? Answer in one word."}],
+            "stream": False,
+            "modalities": ["text"],
+        }
+    )
+    start = time.monotonic()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        futures = [
+            executor.submit(
+                post_chat_completions_raw,
+                host,
+                port,
+                payload_json,
+                timeout_sec=20,
+            )
+            for _ in range(4)
+        ]
+        done, pending = concurrent.futures.wait(
+            futures,
+            timeout=30,
+            return_when=concurrent.futures.ALL_COMPLETED,
+        )
+
+    elapsed = time.monotonic() - start
+    assert not pending, f"[process_kill concurrent] some fault-time requests hung: pending={len(pending)}"
+    assert elapsed < 30, f"[process_kill concurrent] fault-time request convergence is too slow: {elapsed:.2f}s"
+
+    fault_observed = False
+    conc_debug: list[Any] = []
+    for future in done:
+        try:
+            status, body = future.result()
+            conc_debug.append((status, body[:200]))
+            if status >= 500:
+                fault_observed = True
+        except Exception as exc:
+            conc_debug.append(repr(exc))
+            fault_observed = True
+    # DEBUG: remove before merge
+    print(
+        health_final_status,
+        health_final_body[:200],
+        ff_status,
+        ff_body[:200],
+        ff_exc,
+        conc_debug,
+    )
+    assert fault_observed, (
+        "[process_kill concurrent] expected at least one request to fail after process-kill fault injection"
+    )
+
+
+@pytest.mark.slow
+@pytest.mark.skip(reason="issue#2327")
+@hardware_test(res={"cuda": "H100"}, num_cards=2)
+@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_state_converges_after_fault_removed(
+    omni_server_function,
+) -> None:
+    """Black-box: under bounded GPU pressure, a heavy mix chat fails; after hog stops, the same request succeeds.
+
+    Uses a milder sidecar than ``OOM_INJECTION_CONFIG`` (``strict=False`` and a capped
+    ``target_mem_ratio``) so the serving process stays alive: pressure should starve
+    the multimodal forward pass, not exhaust the whole device allocator.
+
+    Fault and recovery phases use raw ``/v1/chat/completions`` HTTP so tests can assert
+    on status codes and JSON error payloads (or fault keywords in streamed bodies).
+    """
+    device_spec = resolve_oom_device_spec(
+        OOM_RECOVER_INJECTION_CONFIG,
+        _stage_config_path_from_omni_server(omni_server_function),
+    )
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_RECOVER_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_RECOVER_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_RECOVER_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_RECOVER_INJECTION_CONFIG["strict"],
+    )
+    host = omni_server_function.host
+    port = omni_server_function.port
+    mix_payload = _mix_chat_completions_probe_payload(omni_server_function)
+    mix_chat_timeout_sec = 180
+    print(
+        "[oom-recover] injection ready "
+        f"device={device_spec} target_mem_ratio={OOM_RECOVER_INJECTION_CONFIG['target_mem_ratio']} "
+        f"strict={OOM_RECOVER_INJECTION_CONFIG['strict']} hold_seconds={OOM_RECOVER_INJECTION_CONFIG['hold_seconds']}"
+    )
+    print(f"[oom-recover] target endpoint http://{host}:{port}/v1/chat/completions")
+
+    failure_observed = False
+    fault_status: int | None = None
+    fault_body = b""
+    try:
+        try:
+            print(f"[oom-recover] fault-phase request start timeout={mix_chat_timeout_sec}s")
+            fault_status, fault_body = post_json_raw(
+                host,
+                port,
+                "/v1/chat/completions",
+                mix_payload,
+                timeout_sec=mix_chat_timeout_sec,
+            )
+            print(f"[oom-recover] fault-phase request done status={fault_status} body_prefix={fault_body[:200]!r}")
+        except Exception as exc:
+            failure_observed = True
+            print(f"[oom-recover] fault-phase request raised {type(exc).__name__}: {exc!r}")
+            assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+        else:
+            assert fault_status is not None
+            failure_observed = fault_status != 200 or _fault_keywords_match_response(
+                status=fault_status,
+                body=fault_body,
+            )
+            assert failure_observed, (
+                "expected mix multimodal request failure while OOM pressure is active; "
+                f"status={fault_status}, body_prefix={fault_body[:500]!r}"
+            )
+            if fault_status >= 400:
+                err = extract_openai_error_contract_from_bytes(fault_body)
+                if err is None:
+                    assert _fault_keywords_match_response(status=fault_status, body=fault_body), (
+                        "non-2xx without OpenAI-style error object should still mention fault hints in body; "
+                        f"status={fault_status}, body_prefix={fault_body[:500]!r}"
+                    )
+                else:
+                    assert isinstance(err.get("message"), str) and err["message"].strip(), (
+                        f"structured error should carry a non-empty message: {err!r}"
+                    )
+            else:
+                assert _fault_keywords_match_response(status=fault_status, body=fault_body), (
+                    "HTTP 200 under pressure should still surface fault hints in the response body; "
+                    f"body_prefix={fault_body[:500]!r}"
+                )
+    finally:
+        print("[oom-recover] stopping OOM sidecar(s)")
+        stop_gpu_oom_hogs(handle)
+        print("[oom-recover] OOM sidecar(s) stopped")
+
+    recovery_deadline = time.monotonic() + 90.0
+    terminal_health: int | None = None
+    unreachable_streak = 0
+    last_health_exc: BaseException | None = None
+    while time.monotonic() < recovery_deadline:
+        try:
+            status, _ = get_health_raw(host, port, timeout_sec=5)
+            unreachable_streak = 0
+            if status in (200, 503):
+                terminal_health = status
+                print(f"[oom-recover] terminal health reached status={terminal_health}")
+                break
+        except Exception as exc:
+            last_health_exc = exc
+            if _looks_like_server_unreachable(exc):
+                unreachable_streak += 1
+                if unreachable_streak >= 5:
+                    pytest.fail(
+                        "after OOM sidecar stopped, /health is unreachable (connection refused / reset). "
+                        "The APIServer process likely exited (e.g. orchestrator thread crash under OOM); "
+                        "this test expects the server to stay up for post-fault health polling. "
+                        f"last_exc={last_health_exc!r}"
+                    )
+            else:
+                unreachable_streak = 0
+        time.sleep(1.0)
+    else:
+        pytest.fail(
+            "server did not converge to a terminal health state after OOM pressure was removed; "
+            f"last_health_exc={last_health_exc!r}"
+        )
+
+    probe_payload = {
+        "model": omni_server_function.model,
+        "messages": [{"role": "user", "content": "What is the capital of China? Answer in one word."}],
+        "stream": False,
+        "modalities": ["text"],
+    }
+    start = time.monotonic()
+    post_mix_status: int | None = None
+    post_mix_body = b""
+    request_status: int | None = None
+    try:
+        assert terminal_health is not None
+        if terminal_health == 200:
+            print(f"[oom-recover] recovery-phase mix request start timeout={mix_chat_timeout_sec}s")
+            post_mix_status, post_mix_body = post_json_raw(
+                host,
+                port,
+                "/v1/chat/completions",
+                mix_payload,
+                timeout_sec=mix_chat_timeout_sec,
+            )
+            print(
+                "[oom-recover] recovery-phase mix request done "
+                f"status={post_mix_status} body_prefix={post_mix_body[:200]!r}"
+            )
+        else:
+            print("[oom-recover] terminal health=503, send lightweight probe request")
+            request_status, _ = post_json_raw(host, port, "/v1/chat/completions", probe_payload, timeout_sec=20)
+            print(f"[oom-recover] lightweight probe done status={request_status}")
+    except Exception:
+        post_mix_status = None
+        request_status = None
+    elapsed = time.monotonic() - start
+    assert elapsed < 200, f"post-fault request should not hang after OOM removal: {elapsed:.2f}s"
+
+    if terminal_health == 200:
+        assert post_mix_status == 200, (
+            "health recovered but repeated mix multimodal request did not return HTTP 200; "
+            f"status={post_mix_status}, body_prefix={post_mix_body[:600]!r}"
+        )
+        recover_err = extract_openai_error_contract_from_bytes(post_mix_body)
+        assert recover_err is None, f"unexpected error object after recovery: {recover_err!r}"
+    else:
+        assert request_status is None or request_status >= 500, (
+            "unhealthy terminal state should fail fast on requests, "
+            f"got health={terminal_health}, request_status={request_status}"
+        )

tests/dfx/reliability/test_reliability_wan22.py

@@ -0,0 +1,411 @@
+"""
+Wan2.2 reliability integration tests.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+import pytest
+import requests
+
+from tests.dfx.conftest import (
+    assert_fault_exception,
+    create_reliability_omni_server_params,
+    resolve_oom_device_spec,
+    supports_video_generation,
+)
+from tests.dfx.reliability.helpers import (
+    get_health_raw,
+    inject_gpu_oom,
+    make_process_kill_fault_injector,
+    stop_gpu_oom_hogs,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.media import generate_synthetic_image
+
+RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
+    {
+        "test_name": "wan22_t2v_reliability_default",
+        "server_params": {
+            "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+            "server_args": [
+                "--num-gpus",
+                "1",
+                "--boundary-ratio",
+                "0.875",
+                "--flow-shift",
+                "5.0",
+                "--disable-log-stats",
+            ],
+        },
+    }
+]
+
+E2E_STAGE_CONFIGS_DIR = Path(__file__).resolve().parent.parent / "e2e" / "stage_configs"
+OOM_INJECTION_CONFIG = {
+    "target_mem_ratio": 0.95,
+    "hold_seconds": 0,
+    "startup_timeout_sec": 20,
+    "strict": True,
+}
+FAULT_ERROR_KEYWORDS = (
+    "oom",
+    "out of memory",
+    "cuda",
+    "job failed",
+    "unknown error",
+    "internal server error",
+    "500 server error",
+)
+PROCESS_KILL_ERROR_KEYWORDS = (
+    "timeout",
+    "did not complete within",
+    "connection",
+    "engine",
+    "orchestrator",
+    "dead",
+    "internal",
+    "500",
+    "503",
+)
+
+WAN_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, E2E_STAGE_CONFIGS_DIR)
+DIFFUSION_VIDEO_PARAMS = [param for param in WAN_PARAMS if supports_video_generation(param.model)]
+
+
+@pytest.mark.slow
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_function, openai_client_function) -> None:
+    stage_config_path = getattr(omni_server_function, "stage_config_path", None)
+    device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_INJECTION_CONFIG["strict"],
+    )
+    try:
+        image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+        request_config = {
+            "form_data": {
+                "prompt": "Generate a realistic road-driving video with camera motion.",
+                "width": 512,
+                "height": 512,
+                "fps": 8,
+                "num_frames": 8,
+                "guidance_scale": 1.0,
+                "flow_shift": 5.0,
+                "num_inference_steps": 4,
+                "seed": 42,
+            },
+            "image_reference": image_data_url,
+            "stream": False,
+        }
+        try:
+            openai_client_function.send_video_diffusion_request(request_config, request_num=1)
+        except Exception as exc:
+            assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+        else:
+            pytest.fail("expected large /v1/videos request failure during GPU OOM injection")
+    finally:
+        stop_gpu_oom_hogs(handle)
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+    "fault_injector",
+    [
+        pytest.param(
+            make_process_kill_fault_injector(
+                grep_patterns="multiprocessing.spawn",
+                signal_name="SIGKILL",
+                limit=1,
+                post_kill_wait_seconds=2.0,
+            ),
+            id="runtime_process_chain",
+        ),
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_video_request_failure(
+    omni_server_after_fault_function,
+    openai_client_function,
+) -> None:
+    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+    request_config = {
+        "form_data": {
+            "prompt": "Generate a realistic road-driving video with camera motion.",
+            "width": 512,
+            "height": 512,
+            "fps": 8,
+            "num_frames": 8,
+            "guidance_scale": 1.0,
+            "flow_shift": 5.0,
+            "num_inference_steps": 4,
+            "seed": 42,
+        },
+        "image_reference": image_data_url,
+        "stream": False,
+    }
+    try:
+        openai_client_function.send_video_diffusion_request(request_config, request_num=1)
+    except Exception as exc:
+        assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+    else:
+        pytest.fail("expected /v1/videos request failure after process-kill injection")
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+    "fault_injector",
+    [
+        pytest.param(
+            make_process_kill_fault_injector(
+                grep_patterns="multiprocessing.spawn",
+                signal_name="SIGKILL",
+                limit=1,
+                post_kill_wait_seconds=2.0,
+            ),
+            id="runtime_process_chain",
+        ),
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_video_health_fast_fail_and_concurrent(
+    omni_server_after_fault_function,
+    openai_client_function,
+) -> None:
+    """Black-box: after process kill, /v1/videos fails fast and concurrent calls don't hang.
+
+    /health→503 is checked last: optional ``pytest.skip`` then ``assert`` (drop the ``if`` to harden).
+    """
+    host = omni_server_after_fault_function.host
+    port = omni_server_after_fault_function.port
+    url = f"http://{host}:{port}/v1/videos"
+    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}"
+    request_config = {
+        "form_data": {
+            "prompt": "Generate a realistic road-driving video with camera motion.",
+            "width": 512,
+            "height": 512,
+            "fps": 8,
+            "num_frames": 8,
+            "guidance_scale": 1.0,
+            "flow_shift": 5.0,
+            "num_inference_steps": 4,
+            "seed": 42,
+        },
+        "image_reference": image_data_url,
+        "stream": False,
+    }
+
+    # Phase-1: one new /v1/videos request should fail fast.
+    payload = {
+        "prompt": "fast-fail check",
+        "width": "512",
+        "height": "512",
+        "fps": "8",
+        "num_frames": "8",
+        "num_inference_steps": "4",
+    }
+    start = time.monotonic()
+    try:
+        response = requests.post(
+            url,
+            data=payload,
+            headers={"Accept": "application/json"},
+            timeout=20,
+        )
+        elapsed = time.monotonic() - start
+        assert elapsed < 15, f"[process_kill fast_fail] /v1/videos did not fail fast after fault: {elapsed:.2f}s"
+        assert response.status_code >= 500, (
+            "[process_kill fast_fail] expected server-side error after fatal fault, "
+            f"got status={response.status_code}, body={response.text[:200]!r}"
+        )
+    except Exception:
+        elapsed = time.monotonic() - start
+        assert elapsed < 15, f"[process_kill fast_fail] /v1/videos exception was too slow after fault: {elapsed:.2f}s"
+
+    # Phase-2: concurrent requests should converge and at least one should fail.
+    start = time.monotonic()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+        futures = [
+            executor.submit(openai_client_function.send_video_diffusion_request, request_config, 1) for _ in range(3)
+        ]
+        done, pending = concurrent.futures.wait(
+            futures,
+            timeout=40,
+            return_when=concurrent.futures.ALL_COMPLETED,
+        )
+
+    elapsed = time.monotonic() - start
+    assert not pending, f"[process_kill concurrent] some fault-time video requests hung: pending={len(pending)}"
+    assert elapsed < 40, f"[process_kill concurrent] fault-time video request convergence is too slow: {elapsed:.2f}s"
+
+    fault_observed = False
+    conc_debug: list[Any] = []
+    for future in done:
+        try:
+            future.result()
+            conc_debug.append("ok")
+        except Exception as exc:
+            fault_observed = True
+            conc_debug.append(repr(exc))
+            assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+    assert fault_observed, (
+        "[process_kill concurrent] expected at least one /v1/videos request to fail after fault; "
+        f"conc_debug={conc_debug}"
+    )
+
+    # Phase-3: /health→503 after fault. Optional skip today; remove the ``if`` later to harden.
+    deadline = time.monotonic() + 20.0
+    last_observation = ""
+    final_health_status: int | None = None
+    while time.monotonic() < deadline:
+        try:
+            status, body = get_health_raw(host, port, timeout_sec=5)
+            last_observation = f"http={status}, body={body[:200]!r}"
+            final_health_status = status
+            if status == 503:
+                break
+        except Exception as exc:  # noqa: BLE001
+            last_observation = f"exception={exc!r}"
+        time.sleep(0.5)
+    if final_health_status != 503:
+        pytest.skip("issue#3050")
+    assert final_health_status == 503, (
+        "[process_kill health] expected /health 503 after fatal fault, "
+        f"got status={final_health_status}, last_observation={last_observation}"
+    )
+
+
+@pytest.mark.slow
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+@pytest.mark.skip(reason="issue#2327")
+@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
+def test_reliability_video_oom_recovers_after_fault_removed(
+    omni_server_function,
+) -> None:
+    """Black-box: after removing OOM pressure, service converges to a terminal state and requests do not hang."""
+    stage_config_path = getattr(omni_server_function, "stage_config_path", None)
+    device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
+    handle = inject_gpu_oom(
+        device=device_spec,
+        target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+        hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+        startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+        strict=OOM_INJECTION_CONFIG["strict"],
+    )
+    host = omni_server_function.host
+    port = omni_server_function.port
+    create_url = f"http://{host}:{port}/v1/videos"
+
+    failure_observed = False
+    try:
+        for _ in range(3):
+            try:
+                response = requests.post(
+                    create_url,
+                    data={
+                        "prompt": "oom recover probe",
+                        "width": "512",
+                        "height": "512",
+                        "fps": "8",
+                        "num_frames": "8",
+                        "num_inference_steps": "4",
+                    },
+                    headers={"Accept": "application/json"},
+                    timeout=25,
+                )
+                if response.status_code >= 500:
+                    failure_observed = True
+                    break
+            except Exception:
+                failure_observed = True
+                break
+            time.sleep(1.0)
+    finally:
+        stop_gpu_oom_hogs(handle)
+
+    assert failure_observed, "expected at least one video request failure while OOM pressure is active"
+
+    recovery_deadline = time.monotonic() + 90.0
+    terminal_health: int | None = None
+    unreachable_streak = 0
+    last_health_exc: BaseException | None = None
+    while time.monotonic() < recovery_deadline:
+        try:
+            status, _ = get_health_raw(host, port, timeout_sec=5)
+            unreachable_streak = 0
+            if status in (200, 503):
+                terminal_health = status
+                break
+        except Exception as exc:  # noqa: BLE001
+            last_health_exc = exc
+            msg = str(exc).lower()
+            if "connection refused" in msg or "actively refused" in msg:
+                unreachable_streak += 1
+                if unreachable_streak >= 5:
+                    pytest.fail(
+                        "after OOM sidecar stopped, /health is unreachable repeatedly; "
+                        "the APIServer process likely exited unexpectedly "
+                        f"(last_exc={last_health_exc!r})"
+                    )
+            else:
+                unreachable_streak = 0
+        time.sleep(1.0)
+    else:
+        pytest.fail(
+            "wan22 server did not converge to terminal health (200/503) after OOM pressure was removed; "
+            f"last_health_exc={last_health_exc!r}"
+        )
+
+    assert terminal_health is not None
+    start = time.monotonic()
+    recovery_resp: requests.Response | None = None
+    try:
+        recovery_resp = requests.post(
+            create_url,
+            data={
+                "prompt": "post-recovery admission check",
+                "width": "512",
+                "height": "512",
+                "fps": "8",
+                "num_frames": "8",
+                "num_inference_steps": "4",
+            },
+            headers={"Accept": "application/json"},
+            timeout=30,
+        )
+        recovery_status = recovery_resp.status_code
+        recovery_text_prefix = recovery_resp.text[:300]
+    except Exception as exc:  # noqa: BLE001
+        recovery_status = None
+        recovery_text_prefix = repr(exc)
+    elapsed = time.monotonic() - start
+    assert elapsed < 30, f"post-fault /v1/videos should not hang after OOM removal: {elapsed:.2f}s"
+
+    if terminal_health == 200:
+        assert recovery_status == 200, (
+            "health recovered but /v1/videos admission did not succeed; "
+            f"status={recovery_status}, body={recovery_text_prefix!r}"
+        )
+        assert recovery_resp is not None
+        recovery_payload = recovery_resp.json()
+        assert "id" in recovery_payload, f"post-recovery create payload missing id: {recovery_payload!r}"
+    else:
+        assert recovery_status is None or recovery_status >= 500, (
+            "unhealthy terminal state should fail fast on /v1/videos, "
+            f"got health={terminal_health}, request_status={recovery_status}, body={recovery_text_prefix!r}"
+        )

tests/diffusion/cache/test_teacache_extractors.py

@@ -13,6 +13,8 @@ used by TeaCache. Each model's extractor can be tested by:
 
 Currently implemented:
 - TestFlux2KleinExtractor: Flux2Klein model extractor
+- TestFlux2Extractor: Flux2 model extractor
+- TestFluxExtractor: Flux model extractor
 """
 
 from abc import ABC, abstractmethod
@@ -22,7 +24,12 @@ import pytest
 import torch
 
 from tests.helpers.mark import hardware_test
-from vllm_omni.diffusion.cache.teacache.extractors import extract_flux2_context, extract_flux2_klein_context
+from vllm_omni.diffusion.cache.teacache.extractors import (
+    extract_flux2_context,
+    extract_flux2_klein_context,
+    extract_flux_context,
+)
+from vllm_omni.diffusion.models.flux.flux_transformer import FluxTransformer2DModel
 from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import (
     Flux2Transformer2DModel,
 )
@@ -234,7 +241,7 @@ class TestFlux2Extractor(BaseExtractorTest):
         in_channels (128) to inner_dim (6144), so modulated_input should match
         the projected shape, not the input shape.
         """
-        context = extract_flux2_klein_context(flux2_module, **sample_inputs)
+        context = extract_flux2_context(flux2_module, **sample_inputs)
 
         batch_size, img_seq_len, _ = sample_inputs["hidden_states"].shape
         inner_dim = flux2_module.inner_dim
@@ -277,3 +284,139 @@ class TestFlux2Extractor(BaseExtractorTest):
                 img_ids=torch.randint(0, 64, (1, 1024, 4)),
                 txt_ids=torch.randint(0, 64, (1, 512, 4)),
             )
+
+
+@pytest.mark.cpu
+class TestFluxExtractor(BaseExtractorTest):
+    """Test extract_flux_context function."""
+
+    @pytest.fixture(autouse=True)
+    def cpu_vllm_config(self):
+        """Force CPU custom-op dispatch for this test class."""
+        from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
+
+        with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
+            yield
+
+    @pytest.fixture(autouse=True)
+    def mock_flux_attention_backend(self):
+        """Use the SDPA backend so FLUX can be instantiated in CPU tests."""
+        from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
+
+        with patch("vllm_omni.diffusion.attention.layer.get_attn_backend", return_value=SDPABackend):
+            yield
+
+    def get_extractor(self):
+        return extract_flux_context
+
+    @pytest.fixture
+    def flux_module(self):
+        """Create a minimal FluxTransformer2DModel for testing."""
+        return FluxTransformer2DModel(
+            num_layers=2,
+            num_single_layers=2,
+            num_attention_heads=2,
+            attention_head_dim=16,
+            joint_attention_dim=32,
+            pooled_projection_dim=16,
+            axes_dims_rope=(4, 4, 8),
+        )
+
+    @pytest.fixture
+    def flux_module_without_guidance(self):
+        """Create a minimal non-guidance-distilled FLUX transformer."""
+        return FluxTransformer2DModel(
+            num_layers=2,
+            num_single_layers=2,
+            num_attention_heads=2,
+            attention_head_dim=16,
+            joint_attention_dim=32,
+            pooled_projection_dim=16,
+            guidance_embeds=False,
+            axes_dims_rope=(4, 4, 8),
+        )
+
+    def get_module(self, flux_module):
+        return flux_module
+
+    @pytest.fixture
+    def sample_inputs(self):
+        """Create sample input tensors for Flux."""
+        batch_size = 1
+        img_seq_len = 16
+        txt_seq_len = 8
+        in_channels = 64  # Flux default in_channels
+        txt_dim = 32
+        pooled_dim = 16
+
+        return {
+            "hidden_states": torch.randn(batch_size, img_seq_len, in_channels),
+            "encoder_hidden_states": torch.randn(batch_size, txt_seq_len, txt_dim),
+            "pooled_projections": torch.randn(batch_size, pooled_dim),
+            "timestep": torch.tensor([500]),
+            "img_ids": torch.randint(0, 64, (batch_size, img_seq_len, 3)),
+            "txt_ids": torch.randint(0, 64, (batch_size, txt_seq_len, 3)),
+            "guidance": torch.tensor([3.5]),
+        }
+
+    def get_sample_inputs(self, sample_inputs):
+        return sample_inputs
+
+    def test_modulated_input_shape(self, flux_module, sample_inputs):
+        """Test that modulated_input has the projected FLUX inner dimension."""
+        context = extract_flux_context(flux_module, **sample_inputs)
+
+        batch_size, img_seq_len, _ = sample_inputs["hidden_states"].shape
+        assert context.modulated_input.shape == (batch_size, img_seq_len, flux_module.inner_dim)
+
+    def test_run_transformer_blocks_callable(self, flux_module, sample_inputs):
+        """Test that run_transformer_blocks is callable."""
+        context = extract_flux_context(flux_module, **sample_inputs)
+        assert callable(context.run_transformer_blocks)
+
+    def test_postprocess_callable(self, flux_module, sample_inputs):
+        """Test that postprocess is callable."""
+        context = extract_flux_context(flux_module, **sample_inputs)
+        assert callable(context.postprocess)
+
+    def test_postprocess_output_shape(self, flux_module, sample_inputs):
+        """Test that postprocess projects back to the input channel width."""
+        context = extract_flux_context(flux_module, **sample_inputs)
+        output = context.postprocess(context.hidden_states)
+
+        assert output.sample.shape == sample_inputs["hidden_states"].shape
+
+    def test_postprocess_return_tuple_when_return_dict_false(self, flux_module, sample_inputs):
+        """Test that postprocess honors return_dict=False."""
+        context = extract_flux_context(flux_module, **sample_inputs, return_dict=False)
+        output = context.postprocess(context.hidden_states)
+
+        assert isinstance(output, tuple)
+        assert len(output) == 1
+        assert output[0].shape == sample_inputs["hidden_states"].shape
+
+    def test_without_guidance(self, flux_module_without_guidance, sample_inputs):
+        """Test context extraction works for FLUX variants without guidance embeddings."""
+        inputs = sample_inputs.copy()
+        inputs["guidance"] = None
+
+        context = extract_flux_context(flux_module_without_guidance, **inputs)
+
+        assert context is not None
+        assert context.temb is not None
+
+    def test_invalid_module_raises_error(self):
+        """Test that invalid module without transformer_blocks raises ValueError."""
+        invalid_module = Mock()
+        invalid_module.transformer_blocks = []
+
+        with pytest.raises(ValueError, match="Module must have transformer_blocks"):
+            extract_flux_context(
+                invalid_module,
+                hidden_states=torch.randn(1, 16, 64),
+                encoder_hidden_states=torch.randn(1, 8, 32),
+                pooled_projections=torch.randn(1, 16),
+                timestep=torch.tensor([500]),
+                img_ids=torch.randint(0, 64, (1, 16, 3)),
+                txt_ids=torch.randint(0, 64, (1, 8, 3)),
+            )

tests/diffusion/test_diffusion_engine_metrics.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Source-level regression tests for DiffusionEngine.
+
+These tests verify naming conventions and patterns by inspecting source code
+at the function level using AST. They are intentionally coupled to the source
+layout and should be updated whenever the metrics construction code is
+refactored.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+
+_ENGINE_PATH = os.path.normpath(
+    os.path.join(
+        os.path.dirname(__file__),
+        os.pardir,
+        os.pardir,
+        "vllm_omni",
+        "diffusion",
+        "diffusion_engine.py",
+    )
+)
+
+
+def _read_engine_source() -> str:
+    with open(_ENGINE_PATH) as f:
+        return f.read()
+
+
+def _get_function_source(source: str, class_name: str | None, func_name: str) -> str:
+    """Extract the source of a specific function/method using AST.
+
+    Args:
+        source: Full file source code.
+        class_name: Enclosing class name, or None for module-level functions.
+        func_name: Function/method name.
+
+    Returns:
+        Source code of the function body.
+    """
+    tree = ast.parse(source)
+    if class_name is not None:
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef) and node.name == class_name:
+                for item in node.body:
+                    if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) and item.name == func_name:
+                        result = ast.get_source_segment(source, item)
+                        assert result is not None, f"{class_name}.{func_name} source not found"
+                        return result
+    else:
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name:
+                result = ast.get_source_segment(source, node)
+                assert result is not None, f"{func_name} source not found"
+                return result
+    raise AssertionError(f"Function {class_name + '.' if class_name else ''}{func_name} not found in source")
+
+
+class TestMetricKeys:
+    """Verify metric naming conventions in DiffusionEngine.step()."""
+
+    def test_no_duplicate_preprocess_key(self) -> None:
+        """step() should not contain 'preprocessing_time_ms' (duplicate of 'preprocess_time_ms')."""
+        source = _read_engine_source()
+        step_source = _get_function_source(source, "DiffusionEngine", "step")
+        assert "preprocessing_time_ms" not in step_source, (
+            "Found duplicate key 'preprocessing_time_ms' in step() — should only use 'preprocess_time_ms'"
+        )
+
+    def test_metric_key_naming_consistency(self) -> None:
+        """In step(): exec_time should use exec_total_time, total_time should use step_total_ms."""
+        source = _read_engine_source()
+        step_source = _get_function_source(source, "DiffusionEngine", "step")
+        lines = step_source.split("\n")
+
+        found_exec = False
+        found_total = False
+        for line in lines:
+            if '"diffusion_engine_exec_time_ms"' in line:
+                found_exec = True
+                assert "exec_total_time" in line, (
+                    "diffusion_engine_exec_time_ms should measure executor time only (exec_total_time)"
+                )
+            if '"diffusion_engine_total_time_ms"' in line:
+                found_total = True
+                assert "step_total_ms" in line, (
+                    "diffusion_engine_total_time_ms should measure full step time (step_total_ms)"
+                )
+        assert found_exec, "diffusion_engine_exec_time_ms key not found in step()"
+        assert found_total, "diffusion_engine_total_time_ms key not found in step()"
+
+
+class TestDummyRunAllocation:
+    """Verify _dummy_run generates exact-sized audio arrays."""
+
+    def test_no_oversized_allocation(self) -> None:
+        """_dummy_run should not allocate more audio than needed."""
+        source = _read_engine_source()
+        dummy_source = _get_function_source(source, "DiffusionEngine", "_dummy_run")
+        assert "audio_sr * audio_duration_sec" not in dummy_source, (
+            "_dummy_run should generate exact-sized audio, not allocate and slice"
+        )

tests/distributed/omni_connectors/test_chunk_transfer_adapter.py

@@ -12,6 +12,7 @@ from pytest_mock import MockerFixture
 from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
 from vllm.v1.request import RequestStatus
 
+from vllm_omni.data_entry_keys import OmniPayload
 from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
 from vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter import (
     OmniChunkTransferAdapter,
@@ -111,7 +112,11 @@ def test_load_poll(build_adapter):
     request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1")
 
     adapter.load_async(request)
-    payload = {"code_predictor_codes": [[1]], "hidden_states": torch.tensor([[2.0]]), "finished": True}
+    payload: OmniPayload = {
+        "codes": {"audio": [[1]]},
+        "hidden_states": {"output": torch.tensor([[2.0]])},
+        "meta": {"finished": torch.tensor(True, dtype=torch.bool)},
+    }
     connector.get.return_value = (payload, 16)
     adapter._poll_single_request(request)
 
@@ -154,12 +159,49 @@ def test_send_single_request_cleans_up_after_finished_payload(build_adapter, mon
 def test_update_request_payload(build_adapter):
     adapter, _ = build_adapter()
 
-    adapter._update_request_payload("ext", {"h": torch.tensor([[1.0]]), "codes": [1], "finished": False})
-    merged = adapter._update_request_payload("ext", {"h": torch.tensor([[2.0]]), "codes": [2], "finished": True})
+    first: OmniPayload = {
+        "hidden_states": {"output": torch.tensor([[1.0]])},
+        "codes": {"audio": [1]},
+        "meta": {"finished": torch.tensor(False, dtype=torch.bool)},
+    }
+    adapter._update_request_payload("ext", first)
+    second: OmniPayload = {
+        "hidden_states": {"output": torch.tensor([[2.0]])},
+        "codes": {"audio": [2]},
+        "meta": {"finished": torch.tensor(True, dtype=torch.bool)},
+    }
+    merged = adapter._update_request_payload("ext", second)
 
-    assert torch.equal(merged["h"], torch.tensor([[1.0], [2.0]]))
-    assert merged["codes"] == [1, 2]
-    assert merged["finished"] is True
+    assert torch.equal(merged["hidden_states"]["output"], torch.tensor([[1.0], [2.0]]))
+    assert merged["codes"]["audio"] == [1, 2]
+    assert merged["meta"]["finished"].item() is True
+
+
+def test_load_poll_ar_request_additional_information_concats_tensors(build_adapter):
+    adapter, connector = build_adapter(stage_id=2, model_mode="ar")
+    request = _req("req-merged", RequestStatus.WAITING, external_req_id="ext-merged")
+
+    adapter.request_ids_mapping["req-merged"] = "ext-merged"
+    adapter.request_payload["ext-merged"] = {
+        "hidden_states": {"output": torch.tensor([[1.0]])},
+        "ids": {"prompt": [11, 12]},
+        "meta": {"finished": torch.tensor(False, dtype=torch.bool)},
+    }
+    payload: OmniPayload = {
+        "hidden_states": {"output": torch.tensor([[2.0]])},
+        "meta": {"finished": torch.tensor(True, dtype=torch.bool)},
+    }
+    connector.get.return_value = (payload, 8)
+
+    adapter._poll_single_request(request)
+
+    assert torch.equal(
+        request.additional_information["hidden_states"]["output"],
+        torch.tensor([[1.0], [2.0]]),
+    )
+    # Keys absent from the new chunk are dropped (matches main's behavior).
+    assert "ids" not in request.additional_information
+    assert request.additional_information["meta"]["finished"].item() is True
 
 
 def test_process_and_restore_queues(build_adapter):
@@ -321,7 +363,10 @@ def test_cleanup_after_poll_flow(build_adapter):
     adapter.load_async(request)
 
     adapter.request_ids_mapping["req-flow"] = "ext-flow"
-    payload = {"hidden_states": torch.tensor([[1.0]]), "finished": True}
+    payload: OmniPayload = {
+        "hidden_states": {"output": torch.tensor([[1.0]])},
+        "meta": {"finished": torch.tensor(True, dtype=torch.bool)},
+    }
     connector.get.return_value = (payload, 8)
     adapter._poll_single_request(request)
 

tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py

@@ -17,6 +17,7 @@ import time
 import pytest
 import torch
 
+from tests.helpers.mark import hardware_test
 from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import (
     ManagedBuffer,
     MooncakeTransferEngineConnector,
@@ -24,7 +25,7 @@ from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_c
 )
 
 # All tests in this file require Mooncake TransferEngine and an RDMA environment.
-pytestmark = [pytest.mark.parallel, pytest.mark.gpu]
+pytestmark = [pytest.mark.parallel, pytest.mark.core_model]
 
 # ---------------------------------------------------------------------------
 # Shared helpers
@@ -194,7 +195,7 @@ def _md5(tensor: torch.Tensor) -> str:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestBasicConnector:
     """Verify connector initialization, put, cleanup, and health check."""
 
@@ -254,7 +255,7 @@ class TestBasicConnector:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestEndToEnd:
     """E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types."""
 
@@ -447,7 +448,7 @@ class TestEndToEnd:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestLifecycle:
     """Close, context manager, double-close safety."""
 
@@ -475,9 +476,7 @@ class TestLifecycle:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.cuda
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestGPUPool:
     """GPU memory pool: initialization, put (CPU/GPU tensor), E2E transfer."""
 
@@ -521,12 +520,11 @@ class TestGPUPool:
 
 
 # ---------------------------------------------------------------------------
-# 5. Stress / Correctness tests (marked slow, skipped in quick CI)
+# 5. Stress / Correctness tests, slow
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.slow
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestStressCorrectness:
     """
     Slow but high-value regression tests: concurrent put+get with data

tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py

@@ -340,7 +340,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
     p.add_argument(
         "--max-seed-tts-mean-wer",
         type=float,
-        default=0.02,
+        default=0.5,
         help="If set, fail when seed_tts_content_error_mean is strictly above this value.",
     )
     p.add_argument(

tests/e2e/offline_inference/test_bagel_text2img.py

@@ -304,11 +304,13 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
 @pytest.mark.core_model
 @pytest.mark.advanced_model
 @pytest.mark.diffusion
-@hardware_test(res={"cuda": "H100"})
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_bagel_text2img_mooncake_connector(run_level):
     """Test Bagel text2img with Mooncake connector for inter-stage communication."""
     if not _is_mooncake_master_available():
-        pytest.skip("mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)")
+        raise RuntimeError(
+            "mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)"
+        )
     MOONCAKE_HOST = "127.0.0.1"
     MOONCAKE_RPC_PORT = _find_free_port()
     MOONCAKE_HTTP_PORT = _find_free_port()

tests/e2e/offline_inference/test_internvla_a1.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Offline E2E test for InternVLA-A1 open-loop inference."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from tests.helpers.mark import hardware_test
+
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py"
+
+
+def _required_env(name: str) -> str:
+    value = os.getenv(name)
+    if not value:
+        pytest.skip(f"{name} is not set")
+    return value
+
+
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
+def test_internvla_a1_offline_open_loop(run_level: str) -> None:
+    if run_level != "advanced_model":
+        pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.")
+
+    model_dir = _required_env("INTERNVLA_A1_MODEL_DIR")
+    dataset_dir = _required_env("INTERNVLA_A1_DATASET_DIR")
+    processor_dir = _required_env("INTERNVLA_A1_PROCESSOR_DIR")
+    cosmos_dir = _required_env("INTERNVLA_A1_COSMOS_DIR")
+
+    env = os.environ.copy()
+    env["INTERNVLA_A1_PROCESSOR_DIR"] = processor_dir
+    env["INTERNVLA_A1_COSMOS_DIR"] = cosmos_dir
+
+    with tempfile.TemporaryDirectory(prefix="internvla_a1_e2e_") as tmpdir:
+        output_dir = Path(tmpdir) / "outputs"
+        cmd = [
+            sys.executable,
+            str(EXAMPLE_SCRIPT),
+            "--model-dir",
+            model_dir,
+            "--dataset-dir",
+            dataset_dir,
+            "--output-dir",
+            str(output_dir),
+            "--num-samples",
+            "1",
+            "--num-episodes",
+            "1",
+            "--dtype",
+            "float32",
+            "--attn-implementation",
+            "eager",
+            "--skip-plots",
+        ]
+        subprocess.run(cmd, check=True, env=env)
+
+        summary_path = output_dir / "summary.json"
+        assert summary_path.exists(), f"Missing summary file: {summary_path}"
+
+        with open(summary_path, encoding="utf-8") as f:
+            summary = json.load(f)
+
+        assert summary["mode"] == "registry"
+        assert summary["attn_implementation"] == "eager"
+        assert summary["dtype"] == "float32"
+        assert len(summary["results"]) == 1
+        assert "registry" in summary["eval_summaries"]
+
+        eval_summary = summary["eval_summaries"]["registry"]
+        assert eval_summary["num_episodes"] == 1
+        assert eval_summary["average_mse"] is not None
+        assert eval_summary["average_mae"] is not None
+
+        log_path = output_dir / "registry" / "log.json"
+        assert log_path.exists(), f"Missing evaluation log: {log_path}"

tests/e2e/offline_inference/test_moss_tts_nano.py

@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""E2E offline inference tests for MOSS-TTS-Nano single-stage pipeline."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+from vllm import SamplingParams
+
+from tests.helpers.mark import hardware_test
+from vllm_omni import Omni
+
+MODEL_NAME = "OpenMOSS-Team/MOSS-TTS-Nano"
+SAMPLE_RATE = 48000
+
+DEFAULT_SAMPLING = SamplingParams(
+    temperature=1.0,
+    top_p=1.0,
+    top_k=50,
+    max_tokens=4096,
+    seed=42,
+    detokenize=False,
+)
+
+
+def _build_request(
+    text: str,
+    voice: str = "Junhao",
+    mode: str = "voice_clone",
+    max_new_frames: int = 100,  # short for tests
+    seed: int = 42,
+) -> dict:
+    return {
+        "prompt": "<|im_start|>assistant\n",
+        "additional_information": {
+            "text": [text],
+            "voice": [voice],
+            "mode": [mode],
+            "max_new_frames": [max_new_frames],
+            "seed": [seed],
+        },
+    }
+
+
+def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
+    """Run a single request and return (waveform, sample_rate)."""
+    for stage_outputs in omni.generate(request, DEFAULT_SAMPLING):
+        for req_output in stage_outputs.request_output:
+            mm = req_output.outputs[0].multimodal_output
+            assert mm is not None, "Expected multimodal_output to be non-None"
+            audio = mm.get("audio")
+            sr = mm.get("sr")
+            assert audio is not None, "Expected 'audio' key in multimodal_output"
+            assert isinstance(audio, torch.Tensor), f"audio should be Tensor, got {type(audio)}"
+            return audio.cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
+    raise AssertionError("No stage outputs received")
+
+
+@pytest.fixture(scope="module")
+def omni_engine():
+    """Module-scoped Omni engine to avoid re-loading the model for each test."""
+    return Omni(model=MODEL_NAME, stage_init_timeout=180)
+
+
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"})
+def test_moss_tts_nano_english(omni_engine):
+    """English TTS produces non-empty 48 kHz stereo audio."""
+    req = _build_request("Hello, this is a test of MOSS-TTS-Nano.", voice="Ava")
+    audio, sr = _collect_audio(omni_engine, req)
+
+    assert sr == SAMPLE_RATE, f"Expected sample_rate={SAMPLE_RATE}, got {sr}"
+    assert audio.numel() > 0, "Audio tensor should not be empty"
+    assert not torch.all(audio == 0), "Audio should not be all-zeros (silence)"
+
+
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"})
+def test_moss_tts_nano_chinese(omni_engine):
+    """Chinese TTS produces non-empty audio."""
+    req = _build_request("你好，这是语音合成测试。", voice="Junhao")
+    audio, sr = _collect_audio(omni_engine, req)
+
+    assert sr == SAMPLE_RATE
+    assert audio.numel() > 0
+    assert not torch.all(audio == 0)
+
+
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"})
+def test_moss_tts_nano_deterministic(omni_engine):
+    """Same seed produces identical waveforms."""
+    req = _build_request("Reproducible output test.", voice="Adam", seed=123)
+    audio1, _ = _collect_audio(omni_engine, req)
+    audio2, _ = _collect_audio(omni_engine, req)
+
+    assert audio1.shape == audio2.shape, "Waveform shapes should match with same seed"
+    assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms should match with same seed"
+
+
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"})
+def test_moss_tts_nano_batch(omni_engine):
+    """Batch of two requests returns audio for each."""
+    requests = [
+        _build_request("First request.", voice="Ava"),
+        _build_request("Second request.", voice="Bella"),
+    ]
+    results = []
+    for stage_outputs in omni_engine.generate(requests, [DEFAULT_SAMPLING] * 2):
+        for req_output in stage_outputs.request_output:
+            mm = req_output.outputs[0].multimodal_output
+            assert mm is not None
+            results.append(mm["audio"].cpu())
+
+    assert len(results) == 2, f"Expected 2 outputs, got {len(results)}"
+    for i, audio in enumerate(results):
+        assert audio.numel() > 0, f"Audio {i} is empty"
+
+
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"})
+def test_moss_tts_nano_voice_presets(omni_engine):
+    """Multiple built-in voice presets all produce valid audio."""
+    voices = ["Junhao", "Ava", "Bella"]
+    for voice in voices:
+        req = _build_request(f"Testing voice {voice}.", voice=voice)
+        audio, sr = _collect_audio(omni_engine, req)
+        assert audio.numel() > 0, f"Voice {voice} produced empty audio"
+        assert sr == SAMPLE_RATE

tests/e2e/online_serving/test_bagel_expansion.py

@@ -9,6 +9,7 @@ Coverage:
 - Tensor-Parallel
 - Ulysses-SP
 - Ring-Attention
+- Layerwise Offloading
 
 assert_diffusion_response validates successful generation and the expected
 512x512 resolution.
@@ -31,7 +32,7 @@ PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
 def _get_diffusion_feature_cases(model: str):
     """Return L4 diffusion feature cases for Bagel.
     TeaCache, Cache-DiT, CFG-Parallel, Tensor-Parallel,
-    Ulysses-SP, Ring-Attention.
+    Ulysses-SP, Ring-Attention, Layerwise Offloading.
     """
 
     return [
@@ -111,6 +112,15 @@ def _get_diffusion_feature_cases(model: str):
             id="sp_ring_2",
             marks=PARALLEL_FEATURE_MARKS,
         ),
+        # Layerwise Offloading (single-card)
+        pytest.param(
+            OmniServerParams(
+                model=model,
+                server_args=["--enable-layerwise-offload"],
+            ),
+            id="single_card_layerwise_offload",
+            marks=SINGLE_CARD_FEATURE_MARKS,
+        ),
     ]
 
 
@@ -132,6 +142,7 @@ def test_bagel(
     - Tensor-Parallel (size=2)
     - Ulysses-SP (degree=2)
     - Ring-Attention (degree=2)
+    - Layerwise Offloading
 
     Validation is delegated to assert_diffusion_response in tests/helpers/assertions.py,
     which checks output dimensions and basic correctness.

tests/e2e/online_serving/test_moss_tts_nano.py

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E Online tests for MOSS-TTS-Nano model with text input and audio output.
+
+These tests verify the /v1/audio/speech endpoint works correctly with
+actual model inference, not mocks.
+"""
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+import pytest
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+MODEL = "OpenMOSS-Team/MOSS-TTS-Nano"
+
+
+def get_prompt(prompt_type="text"):
+    """Text prompt for text-to-audio tests."""
+    prompts = {
+        "text": "Hello, this is a test of MOSS-TTS-Nano text to speech synthesis.",
+        "chinese": "你好，这是MOSS-TTS-Nano的语音合成测试。",
+    }
+    return prompts.get(prompt_type, prompts["text"])
+
+
+tts_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL,
+            stage_config_path=get_deploy_config_path("moss_tts_nano.yaml"),
+            server_args=["--disable-log-stats"],
+        ),
+        id="moss_tts_nano",
+    )
+]
+
+
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_server, openai_client) -> None:
+    """
+    Test text input processing and audio output via /v1/audio/speech.
+    Deploy Setting: default yaml
+    Input Modal: text
+    Output Modal: audio (48 kHz)
+    Input Setting: stream=False, voice=Junhao
+    Datasets: single request
+    """
+    request_config = {
+        "model": omni_server.model,
+        "input": get_prompt(),
+        "stream": False,
+        "response_format": "wav",
+        "voice": "Junhao",
+    }
+
+    openai_client.send_audio_speech_request(request_config)
+
+
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_002(omni_server, openai_client) -> None:
+    """
+    Test streaming text-to-audio via /v1/audio/speech.
+    Deploy Setting: default yaml
+    Input Modal: text
+    Output Modal: audio (48 kHz, PCM stream)
+    Input Setting: stream=True, voice=Ava
+    Datasets: single request
+    """
+    request_config = {
+        "model": omni_server.model,
+        "input": get_prompt(),
+        "stream": True,
+        "response_format": "pcm",
+        "voice": "Ava",
+    }
+
+    openai_client.send_audio_speech_request(request_config)
+
+
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_003(omni_server, openai_client) -> None:
+    """
+    Test Chinese text-to-audio via /v1/audio/speech.
+    Deploy Setting: default yaml
+    Input Modal: text (Chinese)
+    Output Modal: audio (48 kHz)
+    Input Setting: stream=False, voice=Junhao
+    Datasets: single request
+    """
+    request_config = {
+        "model": omni_server.model,
+        "input": get_prompt("chinese"),
+        "stream": False,
+        "response_format": "wav",
+        "voice": "Junhao",
+    }
+
+    openai_client.send_audio_speech_request(request_config)

tests/e2e/online_serving/test_qwen2_5_omni_expansion.py

@@ -56,11 +56,7 @@ def get_max_batch_size(size_type="few"):
     return batch_sizes.get(size_type, 5)
 
 
-@pytest.mark.skip(
-    reason="qwen2.5-omni model cannot recognize the synthetic video and audio data, need to update the test data to enable this test case"
-)
 @pytest.mark.advanced_model
-@pytest.mark.core_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -469,7 +469,6 @@ def test_speaker_001(omni_server, openai_client) -> None:
 
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
-@pytest.mark.skip(reason="Known issue: occasional inaccuracy in voice recognition.")
 def test_speaker_002(omni_server, openai_client) -> None:
     """
     Input Modal: text only (one-word answer constraint).

tests/e2e/online_serving/test_voxtral_tts.py

@@ -82,6 +82,30 @@ class TestVoxtralTTSFixedVoice:
             f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
         )
 
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_speech_english_streaming(self, omni_server) -> None:
+        """Test basic streaming English TTS generation."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "input": "Hello, how are you?",
+            "voice": "casual_female",
+            "language": "English",
+            "stream": True,
+            "response_format": "pcm",
+        }
+
+        with httpx.Client(timeout=120.0) as client:
+            with client.stream("POST", url, json=payload) as response:
+                assert response.status_code == 200
+                assert response.headers.get("content-type") == "audio/pcm"
+                total = sum(len(c) for c in response.iter_bytes())
+
+        assert total > MIN_AUDIO_BYTES, (
+            f"Streamed audio too small ({total} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
+        )
+
     @pytest.mark.advanced_model
     @pytest.mark.omni
     @hardware_test(res={"cuda": "H100"}, num_cards=1)

tests/engine/test_output_processor.py

@@ -0,0 +1,211 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression tests for OmniRequestState multimodal DELTA drain and consolidation guard."""
+
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+from vllm.outputs import PoolingRequestOutput
+from vllm.sampling_params import RequestOutputKind
+from vllm.v1.engine import FinishReason
+
+from vllm_omni.engine.output_modality import OutputModalityNames
+from vllm_omni.engine.output_processor import OmniRequestState
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+# Audio is explicitly listed as a drainable modality
+AUDIO = OutputModalityNames.AUDIO
+
+# Latent is explicitly not drainable, but the choice doesn't matter here as
+# long as isn't listed as drainable. I.e., could also be arbitrary keys for
+# the purposes of these tests
+LATENT = OutputModalityNames.LATENT
+
+# NOTE: detokenizer and logprobs aren't really used here, but we mock them since
+# some of the utils called in vLLM superclass assert require them to be None.
+_DETOK = MagicMock(
+    output_token_ids=[0],
+    get_next_output_text=MagicMock(return_value=""),
+    num_output_tokens=MagicMock(return_value=1),
+)
+_LOGPROBS = MagicMock(logprobs=None, cumulative_logprob=None, prompt_logprobs=None)
+
+_DEFAULT_STATE_KWARGS = dict(
+    request_id="r",
+    external_req_id="r",
+    parent_req=None,
+    request_index=0,
+    lora_request=None,
+    prompt=None,
+    prompt_token_ids=[0],
+    prompt_embeds=None,
+    logprobs_processor=_LOGPROBS,
+    detokenizer=_DETOK,
+    max_tokens_param=None,
+    arrival_time=0.0,
+    queue=None,
+    log_stats=False,
+    stream_interval=1,
+)
+
+
+def _make_state(output_kind: RequestOutputKind):
+    return OmniRequestState(**_DEFAULT_STATE_KWARGS, output_kind=output_kind)
+
+
+def test_init_empty_dict():
+    """Ensure mm_accumulated is initially empty."""
+    assert _make_state(RequestOutputKind.CUMULATIVE).mm_accumulated == {}
+    assert _make_state(RequestOutputKind.DELTA).mm_accumulated == {}
+
+
+def test_delta_drains_output_modality_per_step():
+    """DELTA drains the mm_type key (output modality) but preserves hidden-state keys."""
+    s = _make_state(RequestOutputKind.DELTA)
+    audio1, audio2, hs1, hs2 = [torch.ones(num_elem) for num_elem in range(1, 5)]
+
+    # Add audio and hidden state tensors
+    s.add_multimodal_tensor(audio1, mm_type=AUDIO)  # should be drained
+    s.add_multimodal_tensor(hs1, mm_type=LATENT)  # shouldn't be drained
+
+    out1 = s._new_completion_output([1], None, None)
+    out1_audio = out1.multimodal_output[AUDIO]
+    out1_hidden = out1.multimodal_output[LATENT]
+    assert isinstance(out1_audio, torch.Tensor)
+    assert torch.equal(out1.multimodal_output[AUDIO], audio1)
+    assert isinstance(out1_hidden, torch.Tensor)
+    assert torch.equal(out1.multimodal_output[LATENT], hs1)
+
+    # After emission, hidden states should remain, but audio is drained
+    assert set(s.mm_accumulated.keys()) == {LATENT}
+
+    s.add_multimodal_tensor(audio2, AUDIO)
+    s.add_multimodal_tensor(hs2, mm_type=LATENT)
+    out2 = s._new_completion_output([2], None, None)
+    out2_audio = out2.multimodal_output[AUDIO]
+    out2_hidden = out2.multimodal_output[LATENT]
+    assert isinstance(out2_audio, torch.Tensor)
+    assert torch.equal(out2_audio, audio2)
+    # Since hidden isn't drained, it's grown to a list
+    assert isinstance(out2_hidden, list) and len(out2_hidden) == 2
+    assert torch.equal(hs1, out2_hidden[0])
+    assert torch.equal(hs2, out2_hidden[1])
+
+
+def test_cumulative_emits_consolidated_audio_each_step():
+    """Ensure cumulative accumulates and consolidates modality keys every step."""
+    s = _make_state(RequestOutputKind.CUMULATIVE)
+    # NOTE: audio is usually emitted as (1, size) chunks; we need to be sure
+    # to not change the tensor dimension when we consolidate
+    audio1 = torch.ones(1, 500)
+    s.add_multimodal_tensor(audio1, mm_type=AUDIO)
+    req_out = s.make_request_output([1], None, None, None)
+    assert req_out is not None
+    cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
+    # Single chunk keeps original shape [1, 500]
+    assert isinstance(cons_audio, torch.Tensor) and cons_audio.shape == audio1.shape
+
+    audio2 = torch.ones(1, 300)
+    s.add_multimodal_tensor(audio2, mm_type=AUDIO)
+    req_out = s.make_request_output([2], None, None, None)
+    assert req_out is not None
+    cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
+    # After consolidation, audio chunks are concatenated on last axis,
+    # preserving the [1, N] channel dimension
+    total_audio_len = audio1.shape[-1] + audio2.shape[-1]
+    assert isinstance(audio2, torch.Tensor) and cons_audio.shape == (1, total_audio_len)
+
+    assert "audio" in s.mm_accumulated
+
+
+def test_finish_consolidates_hidden_states():
+    """Ensure consolidation merges hidden-state tensor lists on finish."""
+    s = _make_state(RequestOutputKind.CUMULATIVE)
+    s.add_multimodal_tensor(torch.ones(5, 4), mm_type=LATENT)
+    s.add_multimodal_tensor(torch.ones(3, 4), mm_type=LATENT)
+
+    result = s.make_request_output([1], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+
+    hs = result.outputs[0].multimodal_output[LATENT]
+    assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
+
+
+def test_finish_consolidation_for_hs_delta():
+    """Ensure finish doesn't drop the accumulated hidden states."""
+    s = _make_state(RequestOutputKind.DELTA)
+    # hidden state accumulation (nothing drained)
+    s.add_multimodal_tensor({"foo": torch.ones(5, 4)}, mm_type=LATENT)
+    result = s.make_request_output([0], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+    hs = result.outputs[0].multimodal_output["foo"]
+    assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
+
+    # Since we don't drain the hidden states, if we add 3 elements, we should get 8
+    s.add_multimodal_tensor({"foo": torch.ones(3, 4)}, mm_type=LATENT)
+    result = s.make_request_output([0], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+    hs = result.outputs[0].multimodal_output["foo"]
+    assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
+    assert "foo" in s.mm_accumulated
+
+
+def test_finish_consolidation_drains_mm_delta():
+    """Ensure making the request output drains modality deltas (e.g., audio)."""
+    s = _make_state(RequestOutputKind.DELTA)
+    # multimodal data accumulation (drained)
+    s.add_multimodal_tensor({AUDIO: torch.ones(5, 4)}, mm_type=AUDIO)
+    result = s.make_request_output([0], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+    hs = result.outputs[0].multimodal_output[AUDIO]
+    assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
+
+    # Since we did drain the hidden states, we no longer get the 5 back
+    s.add_multimodal_tensor({AUDIO: torch.ones(3, 4)}, mm_type=AUDIO)
+    result = s.make_request_output([0], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+    hs = result.outputs[0].multimodal_output[AUDIO]
+    assert isinstance(hs, torch.Tensor) and hs.shape[0] == 3
+    assert AUDIO not in s.mm_accumulated  # drained
+
+
+@pytest.mark.parametrize("mm_type", [AUDIO, "hidden"])
+def test_final_only_consolidates_drainable_keys(mm_type):
+    """FINAL_ONLY never drains per-step, so modality keys and hidden state
+    keys both accumulate and are consolidated on finish."""
+    s = _make_state(RequestOutputKind.FINAL_ONLY)
+
+    # NOTE: Currently there is brittlness in the tensor stacking, so we just
+    # test a 1D tensor here. The intention is just to ensure audio /hidden
+    # behave the same.
+    s.add_multimodal_tensor(torch.ones(500), mm_type=mm_type)
+    # Non-finish step returns None without calling _new_completion_output
+    assert s.make_request_output([1], None, None, None) is None
+    assert mm_type in s.mm_accumulated
+
+    s.add_multimodal_tensor(torch.ones(300), mm_type=mm_type)
+    result = s.make_request_output([2], None, FinishReason.STOP, None)
+    assert result is not None and not isinstance(result, PoolingRequestOutput)
+
+    audio = result.outputs[0].multimodal_output[mm_type]
+    assert isinstance(audio, torch.Tensor)
+    assert audio.shape == (800,)
+
+
+def test_cumulative_token_ids_always_set():
+    """cumulative_token_ids is set for all output kinds."""
+    for kind in (RequestOutputKind.DELTA, RequestOutputKind.CUMULATIVE, RequestOutputKind.FINAL_ONLY):
+        s = _make_state(kind)
+        out = s._new_completion_output([42], None, None)
+        assert hasattr(out, "cumulative_token_ids")
+        # The mock detokenizer has output_token_ids=[0]
+        assert list(out.cumulative_token_ids) == [0]
+
+
+def test_cumulative_token_ids_is_a_copy():
+    """cumulative_token_ids must be a snapshot, not a live reference."""
+    s = _make_state(RequestOutputKind.DELTA)
+    out = s._new_completion_output([42], None, None)
+    assert out.cumulative_token_ids is not _DETOK.output_token_ids

tests/entrypoints/openai_api/test_image_server.py

@@ -28,7 +28,6 @@ from vllm_omni.entrypoints.openai.image_api_utils import (
 )
 from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.model_executor.stage_input_processors.glm_image import compute_max_tokens
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
@@ -216,7 +215,11 @@ def async_omni_test_client():
             ]
             default_sampling_params_list = [
                 SamplingParams(temperature=0.1),
-                OmniDiffusionSamplingParams(),
+                OmniDiffusionSamplingParams(
+                    num_inference_steps=4,
+                    guidance_scale=7.5,
+                    generator_device="cpu",
+                ),
             ]
             self.engine = SimpleNamespace(
                 stage_configs=stage_configs,
@@ -576,7 +579,11 @@ def test_multistage_images_async_omni_construction(async_omni_test_client):
 
 
 def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
-    """GLM-Image multistage requests must compute stage-0 max_tokens from size."""
+    """GLM-Image multistage: stage-0 gets target_h/w from requested size.
+
+    max_tokens comes from the deploy YAML default (upper-bound ceiling),
+    NOT computed dynamically from height/width.
+    """
 
     class FakeAsyncOmniClass(AsyncOmni):
         def __init__(self):
@@ -584,8 +591,9 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
                 SimpleNamespace(stage_type="llm", is_comprehension=True, model_arch="GlmImageForConditionalGeneration"),
                 SimpleNamespace(stage_type="diffusion", is_comprehension=False, model_arch="GlmImagePipeline"),
             ]
+            # YAML default max_tokens for GLM-Image AR stage (upper bound for 2048x2048 t2i)
             default_sampling_params_list = [
-                SamplingParams(temperature=0.1, seed=42),
+                SamplingParams(temperature=0.1, seed=42, max_tokens=4353),
                 OmniDiffusionSamplingParams(height=1024, width=1024),
             ]
             self.engine = SimpleNamespace(
@@ -646,7 +654,8 @@ def test_generate_images_async_omni_glm_image_sets_stage0_max_tokens():
     captured = engine.captured_sampling_params_list
     assert captured is not None
     assert len(captured) == 2
-    assert captured[0].max_tokens == compute_max_tokens(1024, 1024, is_i2i=False)
+    # max_tokens comes from YAML default, not computed dynamically
+    assert captured[0].max_tokens == 4353
     assert captured[0].extra_args["target_h"] == 1024
     assert captured[0].extra_args["target_w"] == 1024
     assert captured[1].height == 1024
@@ -1161,9 +1170,10 @@ def test_image_edit_ignores_mock_like_multimodal_limit(async_omni_test_client):
     assert response.status_code == 200
     captured_prompt = engine.captured_prompt
     assert captured_prompt is not None
-    processed_images = captured_prompt["multi_modal_data"]["image"]
-    assert len(processed_images) == 1
-    assert processed_images[0].size == (16, 16)
+    # Multi-stage path uses "img2img" key for single reference image
+    processed_images = captured_prompt["multi_modal_data"]["img2img"]
+    assert isinstance(processed_images, Image.Image)
+    assert processed_images.size == (16, 16)
 
 
 def test_image_edit_parameter_pass(async_omni_test_client):

tests/entrypoints/openai_api/test_serving_chat_multistage_generation.py

@@ -67,6 +67,10 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
     assert len(sampling_params_list) == 3
     assert sampling_params_list[0].temperature == 0.2
     assert sampling_params_list[0].seed == 0
+    assert sampling_params_list[0].extra_args == {"target_h": 768, "target_w": 1024}
+    assert sampling_params_list[1] is not gen_params
+    assert sampling_params_list[2] is not gen_params
+    assert sampling_params_list[1] is not sampling_params_list[2]
     assert sampling_params_list[1].height == 768
     assert sampling_params_list[1].width == 1024
     assert sampling_params_list[1].seed == 0
@@ -75,8 +79,15 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
     assert sampling_params_list[1].num_outputs_per_prompt == 2
     assert sampling_params_list[1].true_cfg_scale == 5.0
     assert sampling_params_list[1].lora_request.name == "adapter-a"
+    assert sampling_params_list[1].lora_scale == 0.6
     assert sampling_params_list[2].height == 768
     assert sampling_params_list[2].width == 1024
+    assert sampling_params_list[2].seed == 0
     assert sampling_params_list[2].num_inference_steps == 28
+    assert sampling_params_list[2].lora_request.name == "adapter-a"
+    assert sampling_params_list[2].lora_scale == 0.6
+    assert gen_params.lora_request is None
     assert engine.default_sampling_params_list[1].height is None
+    assert engine.default_sampling_params_list[1].lora_request is None
     assert engine.default_sampling_params_list[2].resolution == 640
+    assert engine.default_sampling_params_list[2].lora_request is None

tests/entrypoints/openai_api/test_serving_chat_sampling_params.py

@@ -38,7 +38,7 @@ def default_comprehension_params():
         temperature=0.4,
         top_p=0.9,
         top_k=1,
-        max_tokens=2048,
+        max_tokens=4353,
         seed=42,
         repetition_penalty=1.05,
     )
@@ -146,7 +146,7 @@ def test_preserves_yaml_defaults_when_no_request_params(serving_chat, mock_reque
     assert comprehension_params.temperature == 0.4
     assert comprehension_params.top_p == 0.9
     assert comprehension_params.top_k == 1  # YAML custom param preserved
-    assert comprehension_params.max_tokens == 2048
+    assert comprehension_params.max_tokens == 4353
     assert comprehension_params.seed == 42
     assert comprehension_params.repetition_penalty == 1.05  # YAML custom param preserved
 
@@ -190,7 +190,7 @@ def test_max_tokens_uses_yaml_default_when_not_specified(serving_chat, mock_requ
     """Test that max_tokens falls back to YAML default when not in request."""
     result = serving_chat._build_sampling_params_list_from_request(mock_request)
 
-    assert result[0].max_tokens == 2048
+    assert result[0].max_tokens == 4353
 
 
 def test_request_seed_overrides_yaml_default(serving_chat, mock_request):
@@ -588,12 +588,17 @@ class TestResolveHeightWidth:
 
 
 # =============================================================================
-# Tests for _apply_request_overrides with GLM-Image (max_tokens computation)
+# Tests for _apply_request_overrides with GLM-Image (target_h/w injection)
 # =============================================================================
 
 
 class TestApplyRequestOverridesGLMImage:
-    """Test dynamic max_tokens computation for GLM-Image AR stage."""
+    """Test target_h/w injection for GLM-Image AR stage.
+
+    max_tokens is NOT computed dynamically — it comes from the deploy YAML
+    default (e.g. 4353). _apply_request_overrides only injects target_h/w
+    into extra_args so the model can build M-RoPE position grids.
+    """
 
     @pytest.fixture
     def glm_serving_chat(self, mock_engine_client, mocker: MockerFixture):
@@ -601,7 +606,6 @@ class TestApplyRequestOverridesGLMImage:
 
         instance = object.__new__(OmniOpenAIServingChat)
         instance.engine_client = mock_engine_client
-        # Mock the image extraction to return no reference images (t2i by default)
         instance._extract_diffusion_prompt_and_images_from_messages = mocker.MagicMock(return_value=("a cat", []))
         return instance
 
@@ -623,40 +627,41 @@ class TestApplyRequestOverridesGLMImage:
         req.model_fields_set = set()
         return req
 
-    def test_t2i_computes_max_tokens(self, glm_serving_chat, glm_request, default_comprehension_params):
-        """t2i mode: max_tokens computed from height/width, no reference images."""
+    def test_t2i_injects_target_h_w(self, glm_serving_chat, glm_request, default_comprehension_params):
+        """t2i mode: target_h/w injected into extra_args, max_tokens unchanged."""
         result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request)
-        # t2i 1024x1024 = 256 + 1024 + 1 = 1281
-        assert result.max_tokens == 1281
         assert result.extra_args["target_h"] == 1024
         assert result.extra_args["target_w"] == 1024
+        # max_tokens stays at YAML default (not dynamically computed)
+        assert result.max_tokens == 4353
 
-    def test_i2i_computes_fewer_tokens(
+    def test_i2i_injects_target_h_w(
         self, glm_serving_chat, glm_request, default_comprehension_params, mocker: MockerFixture
     ):
-        """i2i mode: max_tokens should be smaller than t2i for same dimensions."""
-        # Make it detect reference images
+        """i2i mode: target_h/w injected, max_tokens unchanged."""
         glm_serving_chat._extract_diffusion_prompt_and_images_from_messages = mocker.MagicMock(
             return_value=("edit this", ["fake_image"])
         )
-
         result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request)
-        # i2i 1024x1024 = 1024 + 1 = 1025
-        assert result.max_tokens == 1025
+        assert result.extra_args["target_h"] == 1024
+        assert result.extra_args["target_w"] == 1024
+        # max_tokens stays at YAML default regardless of t2i/i2i
+        assert result.max_tokens == 4353
 
-    def test_dynamic_max_tokens_overrides_user_value(self, glm_serving_chat, glm_request, default_comprehension_params):
-        """When height/width are provided, dynamic computation overrides user max_tokens."""
+    def test_user_max_tokens_preserved(self, glm_serving_chat, glm_request, default_comprehension_params):
+        """User-provided max_tokens is respected (not overridden by dynamic computation)."""
         glm_request.max_tokens = 500
         glm_request.model_fields_set = {"max_tokens"}
 
         result = glm_serving_chat._apply_request_overrides(default_comprehension_params, glm_request)
-        # Dynamic computation from height/width always wins when present
-        assert result.max_tokens == 1281
+        assert result.max_tokens == 500
+        assert result.extra_args["target_h"] == 1024
+        assert result.extra_args["target_w"] == 1024
 
     def test_no_height_width_preserves_default(
         self, glm_serving_chat, mocker: MockerFixture, default_comprehension_params
     ):
-        """When no height/width in extra_body, keep YAML default max_tokens."""
+        """When no height/width in extra_body, keep YAML default max_tokens, no target_h/w."""
         req = mocker.MagicMock()
         req.temperature = None
         req.top_p = None
@@ -673,7 +678,9 @@ class TestApplyRequestOverridesGLMImage:
         req.model_fields_set = set()
 
         result = glm_serving_chat._apply_request_overrides(default_comprehension_params, req)
-        assert result.max_tokens == 2048  # YAML default
+        assert result.max_tokens == 4353  # YAML default
+        # No target_h/w injected when dimensions not provided
+        assert not result.extra_args or "target_h" not in (result.extra_args or {})
 
     def test_size_string_parsed_for_glm_image(
         self, glm_serving_chat, mocker: MockerFixture, default_comprehension_params
@@ -695,5 +702,7 @@ class TestApplyRequestOverridesGLMImage:
         req.model_fields_set = set()
 
         result = glm_serving_chat._apply_request_overrides(default_comprehension_params, req)
-        # 512x512 t2i = 256 + 256 + 1 = 513
-        assert result.max_tokens == 513
+        assert result.extra_args["target_h"] == 512
+        assert result.extra_args["target_w"] == 512
+        # max_tokens stays at YAML default (not dynamically computed)
+        assert result.max_tokens == 4353

tests/entrypoints/openai_api/test_stage_params.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+from vllm import SamplingParams
+
+from vllm_omni.entrypoints.openai.stage_params import (
+    build_stage_sampling_params_list,
+    get_default_sampling_params_list,
+    resolve_stage_sampling_params,
+)
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+
+def test_resolve_stage_sampling_params_clones_stage_default():
+    default = SamplingParams(temperature=0.2, seed=11)
+
+    resolved = resolve_stage_sampling_params(
+        SimpleNamespace(stage_type="llm"),
+        0,
+        [default],
+    )
+
+    assert resolved is not default
+    assert resolved.temperature == 0.2
+    assert resolved.seed == 11
+
+    resolved.seed = 99
+    assert default.seed == 11
+
+
+def test_resolve_stage_sampling_params_uses_diffusion_fallback_when_default_missing():
+    request_params = OmniDiffusionSamplingParams(height=768, width=1024, seed=7)
+
+    resolved = resolve_stage_sampling_params(
+        SimpleNamespace(stage_type="diffusion"),
+        1,
+        [],
+        diffusion_params=request_params,
+    )
+
+    assert resolved is not request_params
+    assert resolved.height == 768
+    assert resolved.width == 1024
+    assert resolved.seed == 7
+
+
+def test_build_stage_sampling_params_list_can_replace_diffusion_defaults():
+    request_params = OmniDiffusionSamplingParams(height=512, width=512)
+    diffusion_default = OmniDiffusionSamplingParams(height=1024, width=1024)
+    llm_default = SamplingParams(temperature=0.1)
+    stages = [
+        SimpleNamespace(stage_type="llm"),
+        SimpleNamespace(stage_type="diffusion"),
+        SimpleNamespace(stage_type="diffusion"),
+    ]
+
+    resolved = build_stage_sampling_params_list(
+        stages,
+        [llm_default, diffusion_default],
+        diffusion_params=request_params,
+        replace_diffusion_params=True,
+    )
+
+    assert resolved[0] is not llm_default
+    assert resolved[0].temperature == 0.1
+    assert resolved[1] is not request_params
+    assert resolved[2] is not request_params
+    assert resolved[1] is not resolved[2]
+    assert resolved[1].height == 512
+    assert resolved[2].height == 512
+    assert resolved[1].width == 512
+    assert resolved[2].width == 512
+
+
+def test_get_default_sampling_params_list_reads_engine_defaults_only():
+    assert get_default_sampling_params_list(SimpleNamespace()) == []
+    assert get_default_sampling_params_list(SimpleNamespace(default_sampling_params_list=None)) == []
+
+    defaults = [SamplingParams(max_tokens=8)]
+    resolved = get_default_sampling_params_list(SimpleNamespace(default_sampling_params_list=defaults))
+    assert resolved == defaults
+    assert resolved is not defaults

tests/entrypoints/test_async_omni.py

@@ -0,0 +1,134 @@
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+from vllm.sampling_params import RequestOutputKind, SamplingParams
+
+from vllm_omni.entrypoints.async_omni import AsyncOmni
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+async def _noop(**kw):
+    pass
+
+
+def get_fake_add_request(submitted_request_ids):
+    async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
+        del prompt, sampling_params_list, final_stage_id, kwargs
+        submitted_request_ids.append(request_id)
+
+    return fake_add_request_async
+
+
+def get_fake_abort(aborted_request_batches):
+    async def fake_abort_async(request_ids):
+        aborted_request_batches.append(list(request_ids))
+
+    return fake_abort_async
+
+
+async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
+    del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
+    if request_id.startswith("cancel-"):
+        await asyncio.Future()
+        return
+    yield SimpleNamespace(
+        stage_id=0,
+        request_output=SimpleNamespace(outputs=[]),
+        finished=True,
+    )
+
+
+def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) -> AsyncOmni:
+    omni = object.__new__(AsyncOmni)
+    omni._pause_cond = asyncio.Condition()
+    omni._paused = False
+    omni.engine = SimpleNamespace(
+        num_stages=1,
+        add_request_async=fake_add_request,
+        abort_async=fake_abort_request,
+    )
+    omni.log_stats = False
+    omni.request_states = {}
+    omni._final_output_handler = lambda: None
+    omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
+    omni._compute_final_stage_id = lambda output_modalities: 0
+    omni._process_orchestrator_results = fake_process_results
+    omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
+    return omni
+
+
+def test_generate_accepts_request_after_repeated_cancellations():
+    async def run_test():
+        submitted_request_ids = []
+        aborted_request_batches = []
+
+        async def collect_outputs(request_id):
+            outputs = []
+            async for output in AsyncOmni.generate(
+                omni,
+                prompt={"prompt": "prompt"},
+                request_id=request_id,
+                sampling_params_list=[SimpleNamespace()],
+                output_modalities=["image"],
+            ):
+                outputs.append(output)
+            return outputs
+
+        omni = get_async_omni_instance(
+            fake_add_request=get_fake_add_request(submitted_request_ids),
+            fake_abort_request=get_fake_abort(aborted_request_batches),
+        )
+
+        assert len(await collect_outputs("baseline")) == 1
+
+        for idx in range(3):
+            task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
+            await asyncio.sleep(0)
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+        assert len(await collect_outputs("after-cancel")) == 1
+        assert submitted_request_ids == [
+            "baseline",
+            "cancel-0",
+            "cancel-1",
+            "cancel-2",
+            "after-cancel",
+        ]
+        assert aborted_request_batches == [
+            ["cancel-0"],
+            ["cancel-1"],
+            ["cancel-2"],
+        ]
+
+    asyncio.run(run_test())
+
+
+@pytest.mark.parametrize(
+    "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY, RequestOutputKind.CUMULATIVE]
+)
+def test_output_kind_is_preserved_with_explicit_sampling_params(output_kind):
+    """Ensure we don't change the output kind in async generate if params are provided directly."""
+
+    captured_params = []
+
+    async def capturing_add_request(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
+        del prompt, final_stage_id, kwargs
+        captured_params.extend(sampling_params_list)
+
+    async def run():
+        omni = get_async_omni_instance(fake_add_request=capturing_add_request)
+        sp = SamplingParams(output_kind=output_kind)
+        async for _ in omni.generate(
+            prompt={"prompt": "test"},
+            request_id="test-req",
+            sampling_params_list=[sp],
+            output_modalities=["text"],
+        ):
+            pass
+
+    asyncio.run(run())
+    assert captured_params[0].output_kind == output_kind

tests/entrypoints/test_async_omni_abort.py

@@ -1,85 +0,0 @@
-import asyncio
-from types import SimpleNamespace
-
-import pytest
-
-from vllm_omni.entrypoints.async_omni import AsyncOmni
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-
-def test_generate_accepts_request_after_repeated_cancellations():
-    async def run_test():
-        submitted_request_ids = []
-        aborted_request_batches = []
-
-        async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
-            del prompt, sampling_params_list, final_stage_id, kwargs
-            submitted_request_ids.append(request_id)
-
-        async def fake_abort_async(request_ids):
-            aborted_request_batches.append(list(request_ids))
-
-        async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
-            del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
-            if request_id.startswith("cancel-"):
-                await asyncio.Future()
-                return
-            yield SimpleNamespace(
-                stage_id=0,
-                request_output=SimpleNamespace(outputs=[]),
-                finished=True,
-            )
-
-        async def collect_outputs(request_id):
-            outputs = []
-            async for output in AsyncOmni.generate(
-                omni,
-                prompt={"prompt": "prompt"},
-                request_id=request_id,
-                sampling_params_list=[SimpleNamespace()],
-                output_modalities=["image"],
-            ):
-                outputs.append(output)
-            return outputs
-
-        omni = object.__new__(AsyncOmni)
-        omni._pause_cond = asyncio.Condition()
-        omni._paused = False
-        omni.engine = SimpleNamespace(
-            num_stages=1,
-            add_request_async=fake_add_request_async,
-            abort_async=fake_abort_async,
-        )
-        omni.log_stats = False
-        omni.request_states = {}
-        omni._final_output_handler = lambda: None
-        omni.resolve_sampling_params_list = lambda params: params
-        omni._compute_final_stage_id = lambda output_modalities: 0
-        omni._process_orchestrator_results = fake_process_results
-        omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
-
-        assert len(await collect_outputs("baseline")) == 1
-
-        for idx in range(3):
-            task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
-            await asyncio.sleep(0)
-            task.cancel()
-            with pytest.raises(asyncio.CancelledError):
-                await task
-
-        assert len(await collect_outputs("after-cancel")) == 1
-        assert submitted_request_ids == [
-            "baseline",
-            "cancel-0",
-            "cancel-1",
-            "cancel-2",
-            "after-cancel",
-        ]
-        assert aborted_request_batches == [
-            ["cancel-0"],
-            ["cancel-1"],
-            ["cancel-2"],
-        ]
-
-    asyncio.run(run_test())

tests/entrypoints/test_utils.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass
 import pytest
 import torch
 from pytest_mock import MockerFixture
+from vllm.sampling_params import RequestOutputKind, SamplingParams
 
 from vllm_omni.config.yaml_util import create_config
 from vllm_omni.diffusion.data import OmniDiffusionConfig
@@ -15,6 +16,7 @@ from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
 from vllm_omni.entrypoints.utils import (
     _convert_dataclasses_to_dict,
     _filter_dict_like_object,
+    coerce_param_message_types,
     filter_dataclass_kwargs,
     load_and_resolve_stage_configs,
     load_stage_configs_from_yaml,
@@ -424,3 +426,44 @@ class TestLoadStageConfigsFromYaml:
 
         assert stages[0]["engine_args"]["nested"]["base"] == 1
         assert stages[0]["engine_args"]["nested"]["override"] == 2
+
+
+class TestCumulativeStreamingCoercion:
+    @pytest.mark.parametrize("skip_clone", [True, False])
+    def test_cumulative_default_becomes_delta_if_stream(self, skip_clone):
+        """Ensure cumulative messages are coercible to delta if streaming."""
+        sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
+        sp.skip_clone = skip_clone
+        result = coerce_param_message_types([sp], is_streaming=True)[0]
+        assert isinstance(result, SamplingParams)
+        assert result.output_kind == RequestOutputKind.DELTA
+        assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
+
+    @pytest.mark.parametrize("skip_clone", [True, False])
+    def test_cumulative_default_becomes_final_only_if_not_stream(self, skip_clone):
+        """Ensure cumulative messages are coercible to final only if not streaming."""
+        sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
+        sp.skip_clone = skip_clone
+        result = coerce_param_message_types([sp], is_streaming=False)[0]
+        assert isinstance(result, SamplingParams)
+        assert result.output_kind == RequestOutputKind.FINAL_ONLY
+        assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
+
+    @pytest.mark.parametrize("is_streaming", [True, False])
+    @pytest.mark.parametrize("output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
+    def test_non_cumulative_are_coerced(self, output_kind, is_streaming):
+        """Ensure non-cumulative params are coerced to the target type."""
+        sp = SamplingParams(output_kind=output_kind)
+        expected = RequestOutputKind.DELTA if is_streaming else RequestOutputKind.FINAL_ONLY
+        result = coerce_param_message_types([sp], is_streaming=is_streaming)[0]
+        assert isinstance(result, SamplingParams)
+        assert result.output_kind == expected
+
+    def test_coercion_applies_to_all_stages(self):
+        """Ensure all stages are coerced to DELTA for streaming."""
+        sp0 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
+        sp1 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
+        result = coerce_param_message_types([sp0, sp1], is_streaming=True)
+        assert all([isinstance(r, SamplingParams) for r in result])
+        assert result[0].output_kind == RequestOutputKind.DELTA
+        assert result[1].output_kind == RequestOutputKind.DELTA

tests/examples/offline_inference/test_text_to_image.py

@@ -27,9 +27,27 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
 
 README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
 
+# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
+_SKIP_T2I_SNIPPET_IDS = frozenset(
+    {
+        "quick_start_002",
+        "more_cli_examples_001",
+        "more_cli_examples_002",
+        "more_cli_examples_003",
+        "more_cli_examples_007",
+        "more_cli_examples_008",
+    }
+)
+_SKIP_T2I_REASON = (
+    "Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
+    "See https://github.com/vllm-project/vllm-omni/issues/3123"
+)
+
 
 @pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
 def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
+    if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
+        pytest.skip(_SKIP_T2I_REASON)
     should_skip, reason = snippet.skip
     if should_skip:
         pytest.skip(reason)

tests/helpers/assertions.py

@@ -318,7 +318,7 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str:
         top = outputs[0]
         label = str(top.get("label", "")).lower()
         conf = float(top.get("score", 0.0))
-        if conf < 0.5:
+        if conf < 0.6:
             gender = "unknown"
         elif ("female" in label) or ("жен" in label):
             gender = "female"

tests/helpers/fixtures/runtime.py

@@ -10,17 +10,27 @@ from __future__ import annotations
 
 import threading
 from collections.abc import Generator
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import pytest
-import yaml
 
-from tests.helpers.runtime import OmniServer
-from tests.helpers.stage_config import modify_stage_config
+if TYPE_CHECKING:
+    from tests.helpers.runtime import OmniServer
 
 omni_fixture_lock = threading.Lock()
 
 
+@pytest.fixture(scope="function")
+def omni_server_function(
+    request: pytest.FixtureRequest,
+    run_level: str,
+    model_prefix: str,
+) -> Generator[OmniServer, Any, None]:
+    from tests.helpers.runtime import run_omni_server
+
+    yield from run_omni_server(request, run_level, model_prefix, omni_fixture_lock)
+
+
 @pytest.fixture(scope="module")
 def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: str) -> Generator[OmniServer, Any, None]:
     """Start vLLM-Omni through the standard or stage-CLI launcher.
@@ -29,81 +39,9 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
     The ``use_stage_cli`` flag on ``OmniServerParams`` routes the setup through the
     stage-CLI harness while still reusing the same fixture grouping semantics.
     """
-    with omni_fixture_lock:
-        from tests.helpers.runtime import OmniServer, OmniServerParams, OmniServerStageCli
+    from tests.helpers.runtime import run_omni_server
 
-        params: OmniServerParams = request.param
-        model = model_prefix + params.model
-        port = params.port
-        stage_config_path = params.stage_config_path
-        if run_level in {"advanced_model", "full_model"} and stage_config_path is not None:
-            with open(stage_config_path, encoding="utf-8") as f:
-                cfg = yaml.safe_load(f) or {}
-            # Strip ``load_format: dummy`` (CI overlay default) so advanced_model
-            # tests use real weights. New schema (``stages:``) writes the field
-            # flat at stage level; legacy schema (``stage_args:``) nests it as
-            # ``engine_args.load_format``. Handle both.
-            new_schema_stages = cfg.get("stages")
-            stage_key = "stages" if new_schema_stages is not None else "stage_args"
-            delete_path = "load_format" if new_schema_stages is not None else "engine_args.load_format"
-            stage_entries = cfg.get(stage_key, [])
-            stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage]
-            stage_config_path = modify_stage_config(
-                stage_config_path,
-                deletes={stage_key: {stage_id: [delete_path] for stage_id in stage_ids}},
-            )
-
-        server_args = params.server_args or []
-        if params.use_omni and params.stage_init_timeout is not None:
-            server_args = [*server_args, "--stage-init-timeout", str(params.stage_init_timeout)]
-        else:
-            server_args = [*server_args, "--stage-init-timeout", "600"]
-        if params.init_timeout is not None:
-            server_args = [*server_args, "--init-timeout", str(params.init_timeout)]
-        else:
-            server_args = [*server_args, "--init-timeout", "900"]
-        if params.use_stage_cli:
-            if not params.use_omni:
-                raise ValueError("omni_server with use_stage_cli=True requires use_omni=True")
-            if stage_config_path is None:
-                raise ValueError("omni_server with use_stage_cli=True requires a stage_config_path")
-            server_args += ["--stage-configs-path", stage_config_path]
-
-            with OmniServerStageCli(
-                model,
-                stage_config_path,
-                server_args,
-                port=port,
-                env_dict=params.env_dict,
-            ) as server:
-                print("OmniServer started successfully")
-                yield server
-                print("OmniServer stopping...")
-        else:
-            if stage_config_path is not None:
-                server_args += ["--stage-configs-path", stage_config_path]
-
-            with (
-                OmniServer(
-                    model,
-                    server_args,
-                    port=port,
-                    env_dict=params.env_dict,
-                    use_omni=params.use_omni,
-                )
-                if port
-                else OmniServer(
-                    model,
-                    server_args,
-                    env_dict=params.env_dict,
-                    use_omni=params.use_omni,
-                )
-            ) as server:
-                print("OmniServer started successfully")
-                yield server
-                print("OmniServer stopping...")
-
-        print("OmniServer stopped")
+    yield from run_omni_server(request, run_level, model_prefix, omni_fixture_lock)
 
 
 @pytest.fixture
@@ -115,6 +53,15 @@ def openai_client(request: pytest.FixtureRequest, run_level: str):
     return OpenAIClientHandler(host=server.host, port=server.port, api_key="EMPTY", run_level=run_level)
 
 
+@pytest.fixture
+def openai_client_function(request: pytest.FixtureRequest, run_level: str):
+    """Resolve ``omni_server_function`` lazily for function-scoped reliability tests."""
+    from tests.helpers.runtime import OpenAIClientHandler
+
+    server = request.getfixturevalue("omni_server_function")
+    return OpenAIClientHandler(host=server.host, port=server.port, api_key="EMPTY", run_level=run_level)
+
+
 @pytest.fixture(scope="module")
 def omni_runner(request: pytest.FixtureRequest, model_prefix: str):
     from tests.helpers.runtime import OmniRunner

tests/helpers/runtime.py

@@ -9,7 +9,9 @@ import socket
 import subprocess
 import sys
 import tempfile
+import threading
 import time
+from collections.abc import Generator
 from dataclasses import dataclass
 from io import BytesIO
 from pathlib import Path
@@ -120,6 +122,89 @@ class OmniServerParams(NamedTuple):
     stage_init_timeout: int | None = None  # None: fixture supplies default (600 s)
 
 
+def run_omni_server(
+    request: Any,
+    run_level: str,
+    model_prefix: str,
+    omni_fixture_lock: threading.Lock,
+) -> Generator["OmniServer", Any, None]:
+    from tests.helpers.stage_config import modify_stage_config
+
+    with omni_fixture_lock:
+        params: OmniServerParams = request.param
+        model = model_prefix + params.model
+        port = params.port
+        stage_config_path = params.stage_config_path
+        if run_level in {"advanced_model", "full_model"} and stage_config_path is not None:
+            with open(stage_config_path, encoding="utf-8") as f:
+                cfg = yaml.safe_load(f) or {}
+            # Strip ``load_format: dummy`` (CI overlay default) so advanced_model
+            # tests use real weights. New schema (``stages:``) writes the field
+            # flat at stage level; legacy schema (``stage_args:``) nests it as
+            # ``engine_args.load_format``. Handle both.
+            new_schema_stages = cfg.get("stages")
+            stage_key = "stages" if new_schema_stages is not None else "stage_args"
+            delete_path = "load_format" if new_schema_stages is not None else "engine_args.load_format"
+            stage_entries = cfg.get(stage_key, [])
+            stage_ids = [stage["stage_id"] for stage in stage_entries if "stage_id" in stage]
+            stage_config_path = modify_stage_config(
+                stage_config_path,
+                deletes={stage_key: {stage_id: [delete_path] for stage_id in stage_ids}},
+            )
+
+        server_args = params.server_args or []
+        if params.use_omni and params.stage_init_timeout is not None:
+            server_args = [*server_args, "--stage-init-timeout", str(params.stage_init_timeout)]
+        else:
+            server_args = [*server_args, "--stage-init-timeout", "600"]
+        if params.init_timeout is not None:
+            server_args = [*server_args, "--init-timeout", str(params.init_timeout)]
+        else:
+            server_args = [*server_args, "--init-timeout", "900"]
+        if params.use_stage_cli:
+            if not params.use_omni:
+                raise ValueError("omni_server with use_stage_cli=True requires use_omni=True")
+            if stage_config_path is None:
+                raise ValueError("omni_server with use_stage_cli=True requires a stage_config_path")
+            server_args += ["--stage-configs-path", stage_config_path]
+
+            with OmniServerStageCli(
+                model,
+                stage_config_path,
+                server_args,
+                port=port,
+                env_dict=params.env_dict,
+            ) as server:
+                print("OmniServer started successfully")
+                yield server
+                print("OmniServer stopping...")
+        else:
+            if stage_config_path is not None:
+                server_args += ["--stage-configs-path", stage_config_path]
+
+            with (
+                OmniServer(
+                    model,
+                    server_args,
+                    port=port,
+                    env_dict=params.env_dict,
+                    use_omni=params.use_omni,
+                )
+                if port
+                else OmniServer(
+                    model,
+                    server_args,
+                    env_dict=params.env_dict,
+                    use_omni=params.use_omni,
+                )
+            ) as server:
+                print("OmniServer started successfully")
+                yield server
+                print("OmniServer stopping...")
+
+        print("OmniServer stopped")
+
+
 class OmniServer:
     """Omniserver for vLLM-Omni tests."""