diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index c2f82519e..8ec4e736c 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -173,8 +173,9 @@ steps: - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: - # 40m (was 20m before the v0.24.0 rebase): cold-cache runs were killed - # at 20m, masking the real failure; healthy runs finish in ~9-15 min. + # 40m (was 20m): nightly history shows passing runs at ~9-12 min, but + # cold-cache runs were killed by the 20m limit (e.g. nightly #2444 + # died at 25.2m wall), masking the real failure output. - | timeout 40m bash -c " export VLLM_LOGGING_LEVEL=DEBUG @@ -202,8 +203,9 @@ steps: - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: - # 40m (was 20m before the v0.24.0 rebase): cold-cache runs were killed - # at 20m, masking the real failure; healthy runs finish in ~9-15 min. + # 40m (was 20m): nightly history shows passing runs at ~9-13 min, but + # cold-cache runs were killed by the 20m limit (e.g. nightly #2444 + # died at 25.2m wall), masking the real failure output. - | timeout 40m bash -c " export VLLM_LOGGING_LEVEL=DEBUG @@ -266,8 +268,9 @@ steps: - vllm_omni/entrypoints/openai/tts_adapters/indextts2.py - vllm_omni/config/pipeline_registry.py - vllm_omni/model_executor/models/registry.py - # 60m/45m (was 20m/20m before the v0.24.0 rebase): the 20m kill fired - # before pytest could report the underlying failure on this 4-GPU step. + # 60m/45m (was 20m/20m): warm runs take a steady ~12.5 min, but the old + # 20m limit killed a cold run at 20.7m (nightly #2444, timed_out) with + # no failure output. timeout_in_minutes: 60 commands: - | @@ -296,8 +299,8 @@ steps: - vllm_omni/model_executor/models/qwen3_omni/ - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml - # 50m (was 30m before the v0.24.0 rebase): multi-stage Qwen3-Omni runs - # exceeded 30m and were killed, masking the real failure. + # 50m (was 30m): nightly history shows warm runs at 15-21 min; 30m left + # no headroom for cold-cache runs, which were killed without output. timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index c0b455b77..90ffcff94 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -12,8 +12,8 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - label: ":full_moon: Omni · Function Test with H100" - # 120m (was 90m before the v0.24.0 rebase): the full_model sweep grew - # past 90m and was killed, masking the real failure. + # 120m (was 90m): nightly runs take 55-70 min, and #2444 (2026-06-29) + # still hit the old 90m ceiling (timed_out at 90.8m) with no output. timeout_in_minutes: 120 commands: - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py @@ -155,8 +155,9 @@ steps: - label: ":full_moon: Omni · Perf Test · No Async Chunk" key: nightly-omni-performance-no-async-chunk - # 300m (was 180m before the v0.24.0 rebase): the no-async-chunk - # benchmark sweep exceeded 180m and was killed mid-run. + # 300m (was 180m): typically 35-44 min, but slow nights blow up badly — + # nightly #2444 (2026-06-29) passed at 179.4m, a hair under the old + # 180m limit; killing such a run would mask the perf result entirely. timeout_in_minutes: 300 commands: - export BENCHMARK_DIR=tests/dfx/perf/results @@ -380,8 +381,8 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Function Test with L4" - # 120m (was 90m before the v0.24.0 rebase): the full_model TTS sweep - # grew past 90m and was killed, masking the real failure. + # 120m (was 90m): passing nightly runs took 72-87 min under the old + # 90m limit (2026-06-23 pass at 86.9m) — effectively zero headroom. timeout_in_minutes: 120 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" @@ -832,8 +833,9 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" - # 120m (was 60m before the v0.24.0 rebase): the X2I function sweep - # exceeded 60m and was killed mid-run. + # 120m (was 60m): nightly runs took 48-58 min under the old 60m limit + # (near-zero headroom) until 2026-06-29; ~18 min since. Revisit if the + # step stays at the lighter weight. timeout_in_minutes: 120 commands: - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy