[CI] Ground timeout rationale comments in measured nightly durations

Replace the generic timeout comments with actual job durations from the
vllm-omni-rebase nightly pipeline (builds #2316-#2486, 2026-06-22..07-01):
Omni H100 function test still timed out at the old 90m ceiling on 06-29;
the no-async-chunk perf test passed at 179.4m against the old 180m limit
the same night; TTS L4 passes ran 72-87m under the old 90m limit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

.buildkite/test-merge.yml

@@ -173,8 +173,9 @@ steps:
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
         commands:
-          # 40m (was 20m before the v0.24.0 rebase): cold-cache runs were killed
-          # at 20m, masking the real failure; healthy runs finish in ~9-15 min.
+          # 40m (was 20m): nightly history shows passing runs at ~9-12 min, but
+          # cold-cache runs were killed by the 20m limit (e.g. nightly #2444
+          # died at 25.2m wall), masking the real failure output.
           - |
             timeout 40m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
@@ -202,8 +203,9 @@ steps:
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
         commands:
-          # 40m (was 20m before the v0.24.0 rebase): cold-cache runs were killed
-          # at 20m, masking the real failure; healthy runs finish in ~9-15 min.
+          # 40m (was 20m): nightly history shows passing runs at ~9-13 min, but
+          # cold-cache runs were killed by the 20m limit (e.g. nightly #2444
+          # died at 25.2m wall), masking the real failure output.
           - |
             timeout 40m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
@@ -266,8 +268,9 @@ steps:
           - vllm_omni/entrypoints/openai/tts_adapters/indextts2.py
           - vllm_omni/config/pipeline_registry.py
           - vllm_omni/model_executor/models/registry.py
-        # 60m/45m (was 20m/20m before the v0.24.0 rebase): the 20m kill fired
-        # before pytest could report the underlying failure on this 4-GPU step.
+        # 60m/45m (was 20m/20m): warm runs take a steady ~12.5 min, but the old
+        # 20m limit killed a cold run at 20.7m (nightly #2444, timed_out) with
+        # no failure output.
         timeout_in_minutes: 60
         commands:
           - |
@@ -296,8 +299,8 @@ steps:
           - vllm_omni/model_executor/models/qwen3_omni/
           - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
           - vllm_omni/deploy/qwen3_omni_moe.yaml
-        # 50m (was 30m before the v0.24.0 rebase): multi-stage Qwen3-Omni runs
-        # exceeded 30m and were killed, masking the real failure.
+        # 50m (was 30m): nightly history shows warm runs at 15-21 min; 30m left
+        # no headroom for cold-cache runs, which were killed without output.
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'

.buildkite/test-nightly.yml

@@ -12,8 +12,8 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
     steps:
       - label: ":full_moon: Omni · Function Test with H100"
-        # 120m (was 90m before the v0.24.0 rebase): the full_model sweep grew
-        # past 90m and was killed, masking the real failure.
+        # 120m (was 90m): nightly runs take 55-70 min, and #2444 (2026-06-29)
+        # still hit the old 90m ceiling (timed_out at 90.8m) with no output.
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
@@ -155,8 +155,9 @@ steps:
 
       - label: ":full_moon: Omni · Perf Test · No Async Chunk"
         key: nightly-omni-performance-no-async-chunk
-        # 300m (was 180m before the v0.24.0 rebase): the no-async-chunk
-        # benchmark sweep exceeded 180m and was killed mid-run.
+        # 300m (was 180m): typically 35-44 min, but slow nights blow up badly —
+        # nightly #2444 (2026-06-29) passed at 179.4m, a hair under the old
+        # 180m limit; killing such a run would mask the perf result entirely.
         timeout_in_minutes: 300
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
@@ -380,8 +381,8 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
     steps:
       - label: ":full_moon: TTS · Function Test with L4"
-        # 120m (was 90m before the v0.24.0 rebase): the full_model TTS sweep
-        # grew past 90m and was killed, masking the real failure.
+        # 120m (was 90m): passing nightly runs took 72-87 min under the old
+        # 90m limit (2026-06-23 pass at 86.9m) — effectively zero headroom.
         timeout_in_minutes: 120
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
@@ -832,8 +833,9 @@ steps:
 
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
-        # 120m (was 60m before the v0.24.0 rebase): the X2I function sweep
-        # exceeded 60m and was killed mid-run.
+        # 120m (was 60m): nightly runs took 48-58 min under the old 60m limit
+        # (near-zero headroom) until 2026-06-29; ~18 min since. Revisit if the
+        # step stays at the lighter weight.
         timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy