[ROCm] [CI] Migrate CI to mi300x for v0.27.x (#5886)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Signed-off-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
TJianandHongsheng Liu authored and GitHub committed 2026-08-14 15:38:47 +08:00
1 parent 2b08419d8b
commit bc73546c06
13 files changed
+403 -233

No files matched your search

+91 -141
View File
@@ -1,153 +1,103 @@
#!/bin/bash
# vllm-omni customized version
# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
# Last synced: 2025-12-15
# Modifications: docker image name for vllm-omni
# This script runs test inside the corresponding ROCm docker container.
set -o pipefail
# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's
# container image is selected by test-template-amd-omni.j2; MI300 has no DinD.
set -euo pipefail
# Export Python path
export PYTHONPATH=".."
: "${PYTHONFAULTHANDLER:=1}"
: "${HF_HOME:=/home/buildkite-agent/huggingface}"
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
: "${HF_HUB_ETAG_TIMEOUT:=60}"
: "${MIOPEN_DEBUG_CONV_DIRECT:=0}"
: "${MIOPEN_DEBUG_CONV_GEMM:=0}"
: "${VLLM_ROCM_USE_AITER:=0}"
export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER
export PYTORCH_ROCM_ARCH=""
export PYTHONPATH="${PYTHONPATH:-..}"
if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then
echo "Error: MI300 CI must run natively with Docker disabled." >&2
exit 1
fi
if [[ -n "${TEST_COMMANDS:-}" ]]; then
commands="${TEST_COMMANDS}"
else
commands="$*"
fi
if [[ -z "${commands}" ]]; then
echo "Error: No test commands provided." >&2
exit 1
fi
echo "--- Native in-pod ROCm CI"
job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
native_cache_root="/tmp/vllm-omni-native-${job_id}"
TMPDIR="${native_cache_root}/tmp"
TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor"
TRITON_CACHE_DIR="${native_cache_root}/triton"
VLLM_CACHE_ROOT="${native_cache_root}/vllm"
XDG_CACHE_HOME="${native_cache_root}/xdg"
HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets"
export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT
export XDG_CACHE_HOME HF_DATASETS_CACHE
mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \
"${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \
"${HF_DATASETS_CACHE}"
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
if ! command -v findmnt >/dev/null 2>&1; then
echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2
exit 1
fi
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2
exit 1
fi
fi
# Print ROCm version
echo "--- ROCm info"
rocminfo
# cleanup older docker images
cleanup_docker() {
# Get Docker's root directory
docker_root=$(docker info -f '{{.DockerRootDir}}')
if [ -z "$docker_root" ]; then
echo "Failed to determine Docker root directory."
exit 1
fi
echo "Docker root directory: $docker_root"
# Check disk usage of the filesystem where Docker's root directory is located
disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
# Define the threshold
threshold=70
if [ "$disk_usage" -gt "$threshold" ]; then
echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
# Remove dangling images (those that are not tagged and not used by any container)
docker image prune -f
# Remove unused volumes / force the system prune for old images as well.
docker volume prune -f && docker system prune --force --filter "until=72h" --all
echo "Docker images and volumes cleanup completed."
else
echo "Disk usage is below $threshold%. No cleanup needed."
fi
}
expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
python3 - <<'PY'
import os
# Call the cleanup docker function
cleanup_docker
import torch
echo "--- Pulling container"
## Temporary change to use AMD Docker Hub to store the vllm-omni image
# to bypass the rate limit issue with ECR Public Gallery.
# Images are now stored in a separate repository for vllm-omni, instead of vllm-ci.
# TODO: @tjtanaa point back to ECR Public Gallery
# once the amd agents are configured to use ECR Public Gallery.
# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
image_name="rocm/vllm-omni:${BUILDKITE_COMMIT}"
container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1"))
assert torch.version.hip, "PyTorch is not a ROCm build"
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
actual = torch.cuda.device_count()
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
PY
# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery.
# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
# sudo apt-get update && sudo apt-get install -y awscli
## Use safe docker login helper to prevent race conditions
# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh"
# safe_docker_login_ecr_public
## Pull the container from AMD Docker Hub
docker pull "${image_name}"
remove_docker_container() {
docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
docker image rm -f "${image_name}" || true
}
trap remove_docker_container EXIT
echo "--- Running container"
HF_CACHE="$(realpath ~)/huggingface"
mkdir -p "${HF_CACHE}"
HF_MOUNT="/root/.cache/huggingface"
if [[ -n "${TEST_COMMAND:-}" ]]; then
commands="$TEST_COMMAND"
else
commands="$@"
fi
echo "Commands:$commands"
PARALLEL_JOB_COUNT=8
MYPYTHONPATH=".."
# Test that we're launching on the machine that has
# proper access to GPUs
render_gid=$(getent group render | cut -d: -f3)
if [[ -z "$render_gid" ]]; then
echo "Error: 'render' group not found. This is required for GPU access." >&2
exit 1
fi
# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
# TODO: @tjtanaa reenable to run VLLM_ROCM_USE_AITER=1 when AITER is shipped with prebuilt kernels.
if [[ $commands == *"--shard-id="* ]]; then
# assign job count as the number of shards used
commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
# assign shard-id for each shard
commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
echo "Shard ${GPU} commands:$commands_gpu"
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
docker run \
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
--network=host \
--shm-size=16gb \
--group-add "$render_gid" \
-e MIOPEN_DEBUG_CONV_DIRECT=0 \
-e MIOPEN_DEBUG_CONV_GEMM=0 \
-e VLLM_ROCM_USE_AITER=0 \
-e HIP_VISIBLE_DEVICES="${GPU}" \
-e HF_TOKEN \
-v "${HF_CACHE}:${HF_MOUNT}" \
-e "HF_HOME=${HF_MOUNT}" \
-e "PYTHONPATH=${MYPYTHONPATH}" \
--name "${container_name}_${GPU}" \
"${image_name}" \
/bin/bash -c "${commands_gpu}" \
|& while read -r line; do echo ">>Shard $GPU: $line"; done &
PIDS+=($!)
done
#wait for all processes to finish and collect exit codes
for pid in "${PIDS[@]}"; do
wait "${pid}"
STATUS+=($?)
done
for st in "${STATUS[@]}"; do
if [[ ${st} -ne 0 ]]; then
echo "One of the processes failed with $st"
exit "${st}"
echo "Commands:${commands}"
if /bin/bash -o pipefail -c '
set -E
test_status=0
trap '\''
command_status=$?
# Keep running subsequent commands, but retain a failure for the final exit.
# Prefer a test failure over pytest "no tests collected" when both occur.
if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then
test_status=${command_status}
fi
done
'\'' ERR
eval "$1"
exit "${test_status}"
' _ "${commands}"; then
exit 0
else
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
docker run \
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
--network=host \
--shm-size=16gb \
--group-add "$render_gid" \
-e MIOPEN_DEBUG_CONV_DIRECT=0 \
-e MIOPEN_DEBUG_CONV_GEMM=0 \
-e VLLM_ROCM_USE_AITER=0 \
-e HF_TOKEN \
-v "${HF_CACHE}:${HF_MOUNT}" \
-e "HF_HOME=${HF_MOUNT}" \
-e "PYTHONPATH=${MYPYTHONPATH}" \
--name "${container_name}" \
"${image_name}" \
/bin/bash -c "${commands}"
exit_code=$?
fi
if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then
echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success."
exit 0
fi
exit "${exit_code}"