From 0ad67f1d8f7b2abc88a5ff65ec0620f502b6803f Mon Sep 17 00:00:00 2001 From: haic0 <149741444+haic0@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:01:31 -0700 Subject: [PATCH] [CI/Build][ROCm] Retry AMD Simple Diffusion shards on GPU hangs (#8520) Signed-off-by: haic0 <149741444+haic0@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: andyluo7 <43718156+andyluo7@users.noreply.github.com> --- .buildkite/amd/test-amd-merge.yml | 7 +++++++ .buildkite/amd/test-amd-ready.yml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 62391a0c6..0df78c98e 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -37,6 +37,13 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking timeout_in_minutes: 45 + # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang. + # It retries 134 once; the Kubernetes job retry starts a fresh pod. + # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable. + retry: + automatic: + - exit_status: 134 + limit: 1 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 245eb2cea..628b3b419 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -37,6 +37,13 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking timeout_in_minutes: 45 + # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang. + # It retries 134 once; the Kubernetes job retry starts a fresh pod. + # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable. + retry: + automatic: + - exit_status: 134 + limit: 1 commands: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM