diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index a06cf96bf..f56f23b5d 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -64,15 +64,22 @@ while true; do done echo "--- Pulling container" -image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni" +## Temporary change to use AMD Docker Hub to store the vllm-ci image +# to bypass the rate limit issue with ECR Public Gallery. +# TODO: @tjtanaa point back to ECR Public Gallery +# once the amd agents are configured to use ECR Public Gallery. +# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni" +image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}-rocm-omni" container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" -# Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images -sudo apt-get update && sudo apt-get install -y awscli -# Use safe docker login helper to prevent race conditions -source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh" -safe_docker_login_ecr_public -# Pull the container from ECR Public Gallery +# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery. +# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images +# sudo apt-get update && sudo apt-get install -y awscli +## Use safe docker login helper to prevent race conditions +# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh" +# safe_docker_login_ecr_public + +## Pull the container from AMD Docker Hub docker pull "${image_name}" @@ -126,8 +133,6 @@ if [[ $commands == *"--shard-id="* ]]; then -e VLLM_ROCM_USE_AITER=0 \ -e HIP_VISIBLE_DEVICES="${GPU}" \ -e HF_TOKEN \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ @@ -160,8 +165,6 @@ else -e MIOPEN_DEBUG_CONV_GEMM=0 \ -e VLLM_ROCM_USE_AITER=0 \ -e HF_TOKEN \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ diff --git a/.buildkite/test-template-amd-omni.j2 b/.buildkite/test-template-amd-omni.j2 index 4612f8ccd..8dc91a117 100644 --- a/.buildkite/test-template-amd-omni.j2 +++ b/.buildkite/test-template-amd-omni.j2 @@ -3,7 +3,7 @@ Last synced: 2025-12-15 Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests #} -{% set docker_image_amd = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT-rocm-omni" %} +{% set docker_image_amd = "rocm/vllm-ci:$BUILDKITE_COMMIT-rocm-omni" %} {% set default_working_dir = "/app/vllm-omni" %} - group: "AMD Tests" @@ -13,7 +13,6 @@ depends_on: ~ soft_fail: false commands: - - "source .buildkite/scripts/docker_login_ecr_public.sh && safe_docker_login_ecr_public" - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ." - "docker push {{ docker_image_amd }}" key: "amd-build" @@ -22,13 +21,15 @@ retry: automatic: - exit_status: -1 # Agent was lost - limit: 1 + limit: 2 - exit_status: -10 # Agent was lost - limit: 1 + limit: 2 + - exit_status: 128 # Git connectivity issues + limit: 2 - exit_status: 1 # Machine occasionally fail limit: 1 agents: - queue: cpu_queue_premerge + queue: amd-cpu {% for step in steps %} {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}