diff --git a/.buildkite/release/release-pipeline.yml b/.buildkite/release/release-pipeline.yml index 351e105cc..80425b928 100644 --- a/.buildkite/release/release-pipeline.yml +++ b/.buildkite/release/release-pipeline.yml @@ -40,9 +40,11 @@ steps: commands: - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh" + # Auto-skip when NIGHTLY=1 (scheduled); otherwise require manual unblock. - block: "Unblock to build release Docker images" depends_on: ~ key: block-build-release-images + if: build.env("NIGHTLY") != "1" - group: "Build release Docker images" key: "build-release-images" @@ -63,7 +65,7 @@ steps: --progress plain \ -f docker/Dockerfile.cuda . - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - # re-tag to default image tag and push, just in case arm64 build fails + # Fallback default tag (no arch suffix) if arm64 build fails. - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" env: @@ -86,6 +88,26 @@ steps: env: DOCKER_BUILDKIT: "1" + # NIGHTLY=1 only: promote ECR images to DockerHub nightly tags, then prune. + - label: "Publish nightly multi-arch image to DockerHub" + depends_on: + - build-release-image-x86 + - build-release-image-arm64 + if: build.env("NIGHTLY") == "1" + key: publish-nightly-images-dockerhub + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/release/scripts/publish-release-images.sh nightly" + - "bash .buildkite/release/scripts/cleanup-nightly-builds.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + - block: "Publish release images to DockerHub" key: block-publish-release-images depends_on: diff --git a/.buildkite/release/scripts/cleanup-nightly-builds.sh b/.buildkite/release/scripts/cleanup-nightly-builds.sh new file mode 100644 index 000000000..7d1838d89 --- /dev/null +++ b/.buildkite/release/scripts/cleanup-nightly-builds.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# +# Clean up old nightly builds from DockerHub (vllm/vllm-omni), keeping the +# newest 14 tags with the nightly- prefix. + +set -ex + +TAG_PREFIX="nightly-" +REPO="vllm/vllm-omni" +REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags" + +echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO" + +if [ -z "$DOCKERHUB_TOKEN" ]; then + echo "Error: DOCKERHUB_TOKEN environment variable is not set" + exit 1 +fi + +if [ -z "$DOCKERHUB_USERNAME" ]; then + echo "Error: DOCKERHUB_USERNAME environment variable is not set" + exit 1 +fi + +echo "Getting DockerHub bearer token..." +set +x +BEARER_TOKEN=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \ + "https://hub.docker.com/v2/users/login" | jq -r '.token') +set -x + +if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then + echo "Error: Failed to get DockerHub bearer token" + exit 1 +fi + +get_all_tags() { + local page=1 + local all_tags="" + # Hub `name` is a partial match (not a strict prefix). Use it to narrow the + # list, then enforce startswith() client-side. Exhaust API pages; do not + # stop merely because a page has no matching tags after filtering. + local name_query + name_query=$(printf '%s' "$TAG_PREFIX" | jq -sRr @uri) + + while true; do + set +x + local response + response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \ + "$REPO_API_URL?page=$page&page_size=100&name=${name_query}") + set -x + + local page_count + page_count=$(echo "$response" | jq -r '.results | length') + if [ -z "$page_count" ] || [ "$page_count" = "0" ] || [ "$page_count" = "null" ]; then + break + fi + + local tags + tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" \ + '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"') + + if [ -n "$tags" ]; then + all_tags="$all_tags$tags"$'\n' + fi + page=$((page + 1)) + done + + echo "$all_tags" | sort -r | cut -d'|' -f2 +} + +delete_tag() { + local tag_name="$1" + echo "Deleting tag: $tag_name" + + local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name" + set +x + local response http_code body + response=$(curl -s -w "\n%{http_code}" -X DELETE \ + -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url") + set -x + http_code=$(printf '%s\n' "$response" | tail -n1) + body=$(printf '%s\n' "$response" | sed '$d') + + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + echo "Successfully deleted tag: $tag_name (HTTP ${http_code})" + return 0 + fi + + local detail="" + if echo "$body" | jq -e '.detail' > /dev/null 2>&1; then + detail=$(echo "$body" | jq -r '.detail') + elif [ -n "$body" ]; then + detail="$body" + else + detail="(empty response body)" + fi + echo "Error: Failed to delete tag $tag_name (HTTP ${http_code}): ${detail}" + return 1 +} + +echo "Fetching all tags from DockerHub..." +all_tags=$(get_all_tags) + +if [ -z "$all_tags" ]; then + echo "No tags found to clean up" + exit 0 +fi + +total_tags=$(echo "$all_tags" | wc -l) +echo "Found $total_tags tags" + +tags_to_keep=14 +tags_to_delete=$((total_tags - tags_to_keep)) + +if [ $tags_to_delete -le 0 ]; then + echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)" + exit 0 +fi + +echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep" + +tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1))) + +if [ -z "$tags_to_delete_list" ]; then + echo "No tags to delete" + exit 0 +fi + +echo "Deleting old tags..." +delete_failures=0 +while IFS= read -r tag; do + if [ -n "$tag" ]; then + if ! delete_tag "$tag"; then + delete_failures=$((delete_failures + 1)) + fi + sleep 1 + fi +done <<< "$tags_to_delete_list" + +if [ "$delete_failures" -gt 0 ]; then + echo "Cleanup finished with ${delete_failures} deletion failure(s)" + exit 1 +fi + +echo "Cleanup completed successfully" diff --git a/.buildkite/release/scripts/publish-release-images.sh b/.buildkite/release/scripts/publish-release-images.sh index e616604ce..56705613b 100755 --- a/.buildkite/release/scripts/publish-release-images.sh +++ b/.buildkite/release/scripts/publish-release-images.sh @@ -1,50 +1,87 @@ #!/bin/bash # SPDX-License-Identifier: Apache-2.0 # -# Publish release Docker images from ECR to DockerHub. -# Pulls per-arch images, tags with `latest` and versioned tags, pushes them, -# then creates and pushes a multi-arch manifest. +# Publish Docker images from ECR staging to DockerHub. +# +# Usage: +# publish-release-images.sh # release mode (default) +# publish-release-images.sh release +# publish-release-images.sh nightly +# +# Release tags: latest, v +# Nightly tags: nightly, nightly- set -euo pipefail -RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//') -if [ -z "${RELEASE_VERSION}" ]; then - echo "ERROR: release-version metadata not set" - exit 1 -fi +MODE="${1:-release}" -COMMIT="$BUILDKITE_COMMIT" DOCKERHUB_REPO="vllm/vllm-omni" ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo" +COMMIT="$BUILDKITE_COMMIT" +PRIMARY_TAGS=() +EXTRA_MANIFEST_TAGS=() + +case "${MODE}" in + release) + RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//') + if [ -z "${RELEASE_VERSION}" ]; then + echo "ERROR: release-version metadata not set" + exit 1 + fi + PRIMARY_TAGS=("latest" "v${RELEASE_VERSION}") + ;; + nightly) + # Commit-pinned manifest reuses nightly-{x86_64,aarch64} arch images. + PRIMARY_TAGS=("nightly") + EXTRA_MANIFEST_TAGS=("nightly-${COMMIT}") + ;; + *) + echo "Usage: $0 {release|nightly}" + exit 2 + ;; +esac echo "========================================" -echo "Publishing release images v${RELEASE_VERSION}" +echo "Publishing ${MODE} images to ${DOCKERHUB_REPO}" echo " Commit: ${COMMIT}" +echo " Primary tags: ${PRIMARY_TAGS[*]}" +if [ ${#EXTRA_MANIFEST_TAGS[@]} -gt 0 ]; then + echo " Extra manifests: ${EXTRA_MANIFEST_TAGS[*]}" +fi echo "========================================" -# Login to ECR to pull staging images aws ecr-public get-login-password --region us-east-1 | \ docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 -docker pull "${ECR_REPO}:${COMMIT}-x86_64" -docker pull "${ECR_REPO}:${COMMIT}-aarch64" +SRC_X86="${ECR_REPO}:${COMMIT}-x86_64" +SRC_ARM="${ECR_REPO}:${COMMIT}-aarch64" -docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64" -docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" -docker push "${DOCKERHUB_REPO}:latest-x86_64" -docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" +docker pull "${SRC_X86}" +docker pull "${SRC_ARM}" -docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64" -docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" -docker push "${DOCKERHUB_REPO}:latest-aarch64" -docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" +publish_manifest() { + local tag="$1" + local arch_tag_base="$2" + docker manifest rm "${DOCKERHUB_REPO}:${tag}" || true + docker manifest create \ + "${DOCKERHUB_REPO}:${tag}" \ + "${DOCKERHUB_REPO}:${arch_tag_base}-x86_64" \ + "${DOCKERHUB_REPO}:${arch_tag_base}-aarch64" + docker manifest push "${DOCKERHUB_REPO}:${tag}" +} -docker manifest rm "${DOCKERHUB_REPO}:latest" || true -docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true -docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64" -docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" -docker manifest push "${DOCKERHUB_REPO}:latest" -docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" +for tag in "${PRIMARY_TAGS[@]}"; do + docker tag "${SRC_X86}" "${DOCKERHUB_REPO}:${tag}-x86_64" + docker tag "${SRC_ARM}" "${DOCKERHUB_REPO}:${tag}-aarch64" + docker push "${DOCKERHUB_REPO}:${tag}-x86_64" + docker push "${DOCKERHUB_REPO}:${tag}-aarch64" + publish_manifest "${tag}" "${tag}" +done + +ARCH_BASE="${PRIMARY_TAGS[0]}" +for tag in "${EXTRA_MANIFEST_TAGS[@]}"; do + publish_manifest "${tag}" "${ARCH_BASE}" +done echo "" -echo "Successfully published release images for v${RELEASE_VERSION}" +echo "Successfully published ${MODE} images"