From 6d39fb25aa6ae5c3820830c6c43bffe77017292c Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 2 Jul 2026 07:54:56 +0000 Subject: [PATCH] [CI] Recalibrate rebase-raised timeouts on post-merge nightly durations Only post-#4709 nightlies (#2474/#2486, both on rebased code) are a valid baseline; the rebase changed step runtimes substantially. Revert the raises that post-merge data no longer supports and keep the ones it confirms: - Omni Function H100: keep 120m (post-merge 59-71 min vs old 90m ceiling) - Omni Perf No Async Chunk: 300m -> 180m (post-merge 41-44 min) - TTS Function L4: 120m -> 90m (post-merge ~27 min) - Diffusion X2I L4: 120m -> 60m (post-merge ~18 min) - IndexTTS2: 60m/45m -> 40m/35m (steady ~12.5 min warm, cold kill at 20.7m) - CustomVoice/Base 40m and Qwen3-Omni 50m: keep (9-21 min warm + cold-cache headroom the old limits lacked) Co-Authored-By: Claude Fable 5 Signed-off-by: tzhouam --- .buildkite/test-merge.yml | 26 +++++++++++++------------- .buildkite/test-nightly.yml | 28 +++++++++++++++------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 8ec4e736c..057a4915b 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -173,9 +173,9 @@ steps: - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: - # 40m (was 20m): nightly history shows passing runs at ~9-12 min, but - # cold-cache runs were killed by the 20m limit (e.g. nightly #2444 - # died at 25.2m wall), masking the real failure output. + # 40m (was 20m): post-rebase runs take 9-12 min warm, but cold-cache + # runs exceed 20m and were killed without failure output (a cold run + # died at 25.2m wall under the old limit). - | timeout 40m bash -c " export VLLM_LOGGING_LEVEL=DEBUG @@ -203,9 +203,9 @@ steps: - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: - # 40m (was 20m): nightly history shows passing runs at ~9-13 min, but - # cold-cache runs were killed by the 20m limit (e.g. nightly #2444 - # died at 25.2m wall), masking the real failure output. + # 40m (was 20m): post-rebase runs take 9-13 min warm, but cold-cache + # runs exceed 20m and were killed without failure output (a cold run + # died at 25.2m wall under the old limit). - | timeout 40m bash -c " export VLLM_LOGGING_LEVEL=DEBUG @@ -268,13 +268,13 @@ steps: - vllm_omni/entrypoints/openai/tts_adapters/indextts2.py - vllm_omni/config/pipeline_registry.py - vllm_omni/model_executor/models/registry.py - # 60m/45m (was 20m/20m): warm runs take a steady ~12.5 min, but the old - # 20m limit killed a cold run at 20.7m (nightly #2444, timed_out) with - # no failure output. - timeout_in_minutes: 60 + # 40m/35m (was 20m/20m): post-rebase runs take a steady ~12.5 min warm, + # but a cold run was killed at 20.7m under the old limit with no + # failure output; the rebase-time raise to 60m/45m was oversized. + timeout_in_minutes: 40 commands: - | - timeout 45m bash -c " + timeout 35m bash -c " export VLLM_LOGGING_LEVEL=DEBUG pytest -s -v tests/e2e/online_serving/test_indextts2.py tests/e2e/offline_inference/test_indextts2.py -m 'advanced_model and cuda' --run-level 'advanced_model' " @@ -299,8 +299,8 @@ steps: - vllm_omni/model_executor/models/qwen3_omni/ - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml - # 50m (was 30m): nightly history shows warm runs at 15-21 min; 30m left - # no headroom for cold-cache runs, which were killed without output. + # 50m (was 30m): post-rebase runs take 15-21 min warm; 30m left no + # headroom for cold-cache runs, which were killed without output. timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 90ffcff94..ddb7960c2 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -12,8 +12,9 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - label: ":full_moon: Omni · Function Test with H100" - # 120m (was 90m): nightly runs take 55-70 min, and #2444 (2026-06-29) - # still hit the old 90m ceiling (timed_out at 90.8m) with no output. + # 120m (was 90m): post-rebase nightlies (#2474/#2486) run 59-71 min — + # too close to the old 90m ceiling, which #2444 already hit (killed at + # 90.8m with no failure output). timeout_in_minutes: 120 commands: - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py @@ -155,10 +156,10 @@ steps: - label: ":full_moon: Omni · Perf Test · No Async Chunk" key: nightly-omni-performance-no-async-chunk - # 300m (was 180m): typically 35-44 min, but slow nights blow up badly — - # nightly #2444 (2026-06-29) passed at 179.4m, a hair under the old - # 180m limit; killing such a run would mask the perf result entirely. - timeout_in_minutes: 300 + # 180m: post-rebase nightlies (#2474/#2486) run 41-44 min; the + # rebase-time raise to 300m targeted a pre-rebase 179.4m outlier and + # is no longer needed. + timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | @@ -381,9 +382,10 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Function Test with L4" - # 120m (was 90m): passing nightly runs took 72-87 min under the old - # 90m limit (2026-06-23 pass at 86.9m) — effectively zero headroom. - timeout_in_minutes: 120 + # 90m: post-rebase nightlies (#2474/#2486) run ~27 min; the rebase-time + # raise to 120m targeted pre-rebase 72-87 min runs and is no longer + # needed. + timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" @@ -833,10 +835,10 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" - # 120m (was 60m): nightly runs took 48-58 min under the old 60m limit - # (near-zero headroom) until 2026-06-29; ~18 min since. Revisit if the - # step stays at the lighter weight. - timeout_in_minutes: 120 + # 60m: post-rebase nightlies (#2474/#2486) run ~18 min; the rebase-time + # raise to 120m targeted pre-rebase 48-58 min runs and is no longer + # needed. + timeout_in_minutes: 60 commands: - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy agents: