From 4d41e84c674c461d633596a296cbab7373d20e25 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sun, 12 Jul 2026 02:50:14 +0800 Subject: [PATCH] ci: raise merge-tier entrypoint timeouts (H100 60->90, L4 30->45) Both entrypoint jobs are killed mid-run by timeout_in_minutes while tests are still passing, not by any failing test. The budgets had no headroom: on main the H100 job timed out in 2 of the last 3 gate builds (2648, 2650) and passed 2651 at 56/60 min; the L4 job passed at 26/30 min only with a warm image cache. On the vLLM 0.25 wheel, model weight loading is slower on the CI runners (like-for-like: +10 min total on H100, 1.7B TTS load 11s -> 132s on L4), which turns the coin flip into a guaranteed timeout (build 2655). Co-Authored-By: Claude Fable 5 Signed-off-by: tzhouam --- .buildkite/test-merge.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 51c4223f2..b2798c458 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -108,7 +108,10 @@ steps: depends_on: upload-merge-pipeline steps: - label: "Entrypoint Test with H100" - timeout_in_minutes: 60 + # The suite relaunches an OmniServer per test class (~30 model loads, + # incl. ten ~27.5 GiB Bagel loads); 60 min passed only ~1 in 3 runs + # even on main, and vLLM 0.25 weight loading needs ~10 min more. + timeout_in_minutes: 90 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" agents: @@ -146,7 +149,9 @@ steps: type: DirectoryOrCreate - label: "Entrypoint Test with L4" - timeout_in_minutes: 30 + # 30 min left no headroom for a cold image pull (~11 min for new base + # layers) plus per-class engine relaunches; main passed at 26/30. + timeout_in_minutes: 45 commands: - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" agents: