diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 62391a0c6..0df78c98e 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -37,6 +37,13 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking timeout_in_minutes: 45 + # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang. + # It retries 134 once; the Kubernetes job retry starts a fresh pod. + # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable. + retry: + automatic: + - exit_status: 134 + limit: 1 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 245eb2cea..628b3b419 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -37,6 +37,13 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking timeout_in_minutes: 45 + # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang. + # It retries 134 once; the Kubernetes job retry starts a fresh pod. + # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable. + retry: + automatic: + - exit_status: 134 + limit: 1 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM