diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index fb703cb45..63433cc12 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -7,17 +7,29 @@ steps: - group: ":card_index_dividers: Simple Test" steps: - - label: "Simple · Diffusion & Model Executor Test" + - label: "Simple · Diffusion & Model Executor Test (1/2)" agent_pool: mi325_1 depends_on: amd-build - timeout_in_minutes: 50 + timeout_in_minutes: 70 mirror_hardwares: [amdproduction] grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + # CPU tests on ROCm gpu machines are very slow, so we give them more time. + - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + + - label: "Simple · Diffusion & Model Executor Test (2/2)" + agent_pool: mi325_1 + depends_on: amd-build + timeout_in_minutes: 70 + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # CPU tests on ROCm gpu machines are very slow, so we give them more time. + - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - label: "Simple · Other Test" agent_pool: mi325_1 @@ -158,7 +170,7 @@ steps: grade: Blocking commands: - | - timeout 20m bash -c ' + timeout 50m bash -c ' export VLLM_LOGGING_LEVEL=DEBUG export VLLM_WORKER_MULTIPROC_METHOD=spawn export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" @@ -170,13 +182,14 @@ steps: depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking + timeout_in_minutes: 50 commands: - | - timeout 30m bash -c ' + timeout 40m bash -c ' export VLLM_LOGGING_LEVEL=DEBUG export VLLM_WORKER_MULTIPROC_METHOD=spawn export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' ' # TODO: Bagel test on ROCm is very unstable. @tjtanaa