From c22019b93a48312e9fafa14382e9b932c0d255ca Mon Sep 17 00:00:00 2001 From: zhumingjue138 Date: Thu, 26 Feb 2026 21:31:36 +0800 Subject: [PATCH] [CI] fixed CI timeout (#1460) Signed-off-by: zhumingjue Signed-off-by: zhumingjue138 --- .buildkite/test-ready.yml | 85 ++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index d7595e26c..3f46336d5 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -2,7 +2,7 @@ steps: - label: "Simple Unit Test" depends_on: upload-ready-pipeline commands: - - "pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml" + - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml" agents: queue: "gpu_1_queue" plugins: @@ -16,12 +16,11 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion Model Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model" + - timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model" agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + queue: "gpu_1_queue" plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT @@ -33,11 +32,13 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion Model CPU offloading Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py - - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py + - | + timeout 20m bash -c ' + pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py && + pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py + ' agents: queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: @@ -51,10 +52,9 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Audio Generation Model Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py + - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py agents: queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: @@ -68,10 +68,9 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion Cache Backend Test" - timeout_in_minutes: 15 depends_on: upload-ready-pipeline commands: - - pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4' + - timeout 15m pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4' agents: queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: @@ -85,10 +84,9 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion Sequence Parallelism Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model + - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -103,10 +101,9 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion GPU Worker Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - pytest -s -v tests/diffusion/test_diffusion_worker.py + - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -121,12 +118,14 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" # - label: "Benchmark & Engine Test with H100" - # timeout_in_minutes: 15 # depends_on: upload-ready-pipeline # commands: - # - export VLLM_WORKER_MULTIPROC_METHOD=spawn - # - pytest -s -v tests/benchmarks/test_serve_cli.py - # - pytest -s -v tests/engine/test_async_omni_engine_abort.py + # - | + # timeout 15m bash -c ' + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/benchmarks/test_serve_cli.py + # pytest -s -v tests/engine/test_async_omni_engine_abort.py + # ' # agents: # queue: "mithril-h100-pool" # plugins: @@ -157,12 +156,14 @@ steps: # type: DirectoryOrCreate - label: "Omni Model Test" - timeout_in_minutes: 17 depends_on: upload-ready-pipeline commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + - | + timeout 17m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + ' agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -176,13 +177,15 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" # - label: "Omni Model Test with H100" - # timeout_in_minutes: 20 # depends_on: upload-ready-pipeline # commands: - # - export VLLM_WORKER_MULTIPROC_METHOD=spawn - # - export VLLM_TEST_CLEAN_GPU_MEMORY="1" - # # - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py - # - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model" + # - | + # timeout 20m bash -c ' + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # export VLLM_TEST_CLEAN_GPU_MEMORY="1" + # # - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py + # pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model" + # ' # agents: # queue: "mithril-h100-pool" # plugins: @@ -213,12 +216,14 @@ steps: # type: DirectoryOrCreate - label: "Qwen3-TTS E2E Test" - timeout_in_minutes: 20 depends_on: upload-ready-pipeline commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py + - | + timeout 20m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py + ' agents: queue: "gpu_4_queue" plugins: @@ -233,11 +238,13 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" # - label: "Diffusion Image Edit Test with H100 (1 GPU)" - # timeout_in_minutes: 20 # depends_on: upload-ready-pipeline # commands: - # - export VLLM_WORKER_MULTIPROC_METHOD=spawn - # - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py + # - | + # timeout 20m bash -c ' + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py + # ' # agents: # queue: "mithril-h100-pool" # plugins: @@ -268,11 +275,13 @@ steps: # type: DirectoryOrCreate # - label: "Bagel Text2Img Model Test with H100" - # timeout_in_minutes: 30 # depends_on: upload-ready-pipeline # commands: - # - export VLLM_WORKER_MULTIPROC_METHOD=spawn - # - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py + # - | + # timeout 30m bash -c ' + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py + # ' # agents: # queue: "mithril-h100-pool" # plugins: