diff --git a/.buildkite/nightly-release-pipeline.yaml b/.buildkite/nightly-release-pipeline.yaml deleted file mode 100644 index bcb84c08d..000000000 --- a/.buildkite/nightly-release-pipeline.yaml +++ /dev/null @@ -1,104 +0,0 @@ -steps: - - label: "Build and upload wheel" - key: "build-wheel" - agents: - queue: cpu_queue_release - commands: - - "curl -LsSf https://astral.sh/uv/install.sh | sh" - - 'export PATH="$HOME/.local/bin:$PATH"' - - "uv venv --python=3.12 && source .venv/bin/activate" - - "uv pip install --upgrade build" - - "python3 -m build" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - - label: "Generate and upload wheel indices" - depends_on: "build-wheel" - allow_dependency_failure: true - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" - - - group: "Build release Docker images" - key: "build-release-images" - steps: - - label: "Build release image - x86_64" - key: build-release-image-x86 - depends_on: ~ - agents: - queue: cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - --build-arg max_jobs=16 \ - --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ - --tag vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m) \ - --tag vllm/vllm-omni:nightly-$(uname -m) \ - --progress plain \ - -f docker/Dockerfile.cuda . - # push to ECR (per-arch, commit-pinned) for staging - - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - # push to DockerHub (per-arch) so the multi-arch manifest can reference them - - "docker push vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m)" - - "docker push vllm/vllm-omni:nightly-$(uname -m)" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Build release image - aarch64" - key: build-release-image-arm64 - depends_on: ~ - agents: - queue: arm64_cpu_queue_release - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - | - DOCKER_BUILDKIT=1 docker build \ - --build-arg max_jobs=16 \ - --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ - --tag vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m) \ - --tag vllm/vllm-omni:nightly-$(uname -m) \ - --progress plain \ - -f docker/Dockerfile.cuda . - # push to ECR (per-arch, commit-pinned) for staging - - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - # push to DockerHub (per-arch) so the multi-arch manifest can reference them - - "docker push vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m)" - - "docker push vllm/vllm-omni:nightly-$(uname -m)" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Publish multi-arch manifest to DockerHub" - key: publish-multi-arch-dockerhub - depends_on: - - build-release-image-x86 - - build-release-image-arm64 - agents: - queue: small_cpu_queue_release - commands: - # ECR multi-arch manifest (commit-pinned, useful for downstream tests) - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" - # DockerHub multi-arch manifests (nightly + commit-pinned) - - "docker manifest create vllm/vllm-omni:nightly vllm/vllm-omni:nightly-x86_64 vllm/vllm-omni:nightly-aarch64 --amend" - - "docker manifest create vllm/vllm-omni:nightly-$BUILDKITE_COMMIT vllm/vllm-omni:nightly-x86_64 vllm/vllm-omni:nightly-aarch64 --amend" - - "docker manifest push vllm/vllm-omni:nightly" - - "docker manifest push vllm/vllm-omni:nightly-$BUILDKITE_COMMIT" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/rebase-pipeline.yaml new file mode 100644 index 000000000..edd98fafd --- /dev/null +++ b/.buildkite/rebase-pipeline.yaml @@ -0,0 +1,38 @@ +steps: + - label: ":docker: Build image" + key: image-build + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." + - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + agents: + queue: "cpu_queue_premerge" + + # L2 Test + - label: "Upload Ready Pipeline" + depends_on: image-build + key: upload-ready-pipeline + commands: + - buildkite-agent pipeline upload .buildkite/test-ready.yml + agents: + queue: "cpu_queue_premerge" + + # L3 Test + - label: "Upload Merge Pipeline" + depends_on: image-build + key: upload-merge-pipeline + commands: + - buildkite-agent pipeline upload .buildkite/test-merge.yml + agents: + queue: "cpu_queue_premerge" + + # L4 Test + - label: "Upload Nightly Pipeline" + depends_on: image-build + key: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" + commands: + - buildkite-agent pipeline upload .buildkite/test-nightly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index edd98fafd..129a59d5b 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -1,38 +1,130 @@ steps: - - label: ":docker: Build image" - key: image-build - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." - - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - agents: - queue: "cpu_queue_premerge" + - input: "Provide Release version here" + id: input-release-version + fields: + - text: "What is the release version?" + key: release-version - # L2 Test - - label: "Upload Ready Pipeline" - depends_on: image-build - key: upload-ready-pipeline - commands: - - buildkite-agent pipeline upload .buildkite/test-ready.yml + - label: "Build and upload wheel" + depends_on: ~ + key: "build-wheel" agents: - queue: "cpu_queue_premerge" + queue: cpu_queue_release + commands: + - "curl -LsSf https://astral.sh/uv/install.sh | sh" + - 'export PATH="$HOME/.local/bin:$PATH"' + - "uv venv --python=3.12 && source .venv/bin/activate" + - "uv pip install --upgrade build" + - "python3 -m build" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" - # L3 Test - - label: "Upload Merge Pipeline" - depends_on: image-build - key: upload-merge-pipeline - commands: - - buildkite-agent pipeline upload .buildkite/test-merge.yml + - label: "Generate and upload wheel indices" + depends_on: "build-wheel" + allow_dependency_failure: true agents: - queue: "cpu_queue_premerge" + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" - # L4 Test - - label: "Upload Nightly Pipeline" - depends_on: image-build - key: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" - commands: - - buildkite-agent pipeline upload .buildkite/test-nightly.yml - agents: - queue: "cpu_queue_premerge" + - block: "Unblock to build release Docker images" + depends_on: ~ + key: block-build-release-images + if: build.env("NIGHTLY") != "1" + + - group: "Build release Docker images" + key: "build-release-images" + depends_on: block-build-release-images + allow_dependency_failure: true + steps: + - label: "Build release image - x86_64" + key: build-release-image-x86 + depends_on: ~ + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + --build-arg max_jobs=16 \ + --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.cuda . + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + # re-tag to default image tag and push, just in case arm64 build fails + - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" + env: + DOCKER_BUILDKIT: "1" + + - label: "Build release image - aarch64" + key: build-release-image-arm64 + depends_on: ~ + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + --build-arg max_jobs=16 \ + --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.cuda . + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + env: + DOCKER_BUILDKIT: "1" + + - group: "Publish release images" + key: "publish-release-images" + steps: + - label: "Create multi-arch manifest" + depends_on: + - build-release-image-x86 + - build-release-image-arm64 + id: create-multi-arch-manifest + agents: + queue: small_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" + + - label: "Publish nightly multi-arch image to DockerHub" + depends_on: + - create-multi-arch-manifest + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/push-nightly-builds.sh" + # Clean up old nightly builds (keep only last 14) + - "bash .buildkite/scripts/cleanup-nightly-builds.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + + - block: "Publish release images to DockerHub" + key: block-publish-release-images + depends_on: + - create-multi-arch-manifest + - input-release-version + if: build.env("NIGHTLY") != "1" + + - label: "Publish release images to DockerHub" + depends_on: + - block-publish-release-images + key: publish-release-images-dockerhub + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/publish-release-images.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh new file mode 100755 index 000000000..ae9571b57 --- /dev/null +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +set -ex + +# Clean up old nightly builds from DockerHub, keeping only the last 14 builds. +# Uses the DockerHub API to list and delete old tags with the given prefix. +# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO] +# Example: cleanup-nightly-builds.sh "nightly-" +# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-omni" + +# Get tag prefix and repo from arguments +TAG_PREFIX="${1:-nightly-}" +REPO="${2:-vllm/vllm-omni}" + +echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO" + +# DockerHub API endpoint for the repository +REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags" + +# Get DockerHub credentials from environment +if [ -z "$DOCKERHUB_TOKEN" ]; then + echo "Error: DOCKERHUB_TOKEN environment variable is not set" + exit 1 +fi + +if [ -z "$DOCKERHUB_USERNAME" ]; then + echo "Error: DOCKERHUB_USERNAME environment variable is not set" + exit 1 +fi + +# Get DockerHub bearer token +echo "Getting DockerHub bearer token..." +set +x +BEARER_TOKEN=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \ + "https://hub.docker.com/v2/users/login" | jq -r '.token') +set -x + +if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then + echo "Error: Failed to get DockerHub bearer token" + exit 1 +fi + +# Function to get all tags from DockerHub +get_all_tags() { + local page=1 + local all_tags="" + + while true; do + set +x + local response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \ + "$REPO_API_URL?page=$page&page_size=100") + set -x + + # Get both last_updated timestamp and tag name, separated by | + local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"') + + if [ -z "$tags" ]; then + break + fi + + all_tags="$all_tags$tags"$'\n' + page=$((page + 1)) + done + + # Sort by timestamp (newest first) and extract just the tag names + echo "$all_tags" | sort -r | cut -d'|' -f2 +} + +delete_tag() { + local tag_name="$1" + echo "Deleting tag: $tag_name" + + local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name" + set +x + local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url") + set -x + + if echo "$response" | jq -e '.detail' > /dev/null 2>&1; then + echo "Warning: Failed to delete tag $tag_name: $(echo "$response" | jq -r '.detail')" + else + echo "Successfully deleted tag: $tag_name" + fi +} + +# Get all nightly- prefixed tags, sorted by last_updated timestamp (newest first) +echo "Fetching all tags from DockerHub..." +all_tags=$(get_all_tags) + +if [ -z "$all_tags" ]; then + echo "No tags found to clean up" + exit 0 +fi + +# Count total tags +total_tags=$(echo "$all_tags" | wc -l) +echo "Found $total_tags tags" + +# Keep only the last 14 builds (including the current one) +tags_to_keep=14 +tags_to_delete=$((total_tags - tags_to_keep)) + +if [ $tags_to_delete -le 0 ]; then + echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)" + exit 0 +fi + +echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep" + +# Get tags to delete (skip the first $tags_to_keep tags) +tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1))) + +if [ -z "$tags_to_delete_list" ]; then + echo "No tags to delete" + exit 0 +fi + +# Delete old tags +echo "Deleting old tags..." +while IFS= read -r tag; do + if [ -n "$tag" ]; then + delete_tag "$tag" + # Add a small delay to avoid rate limiting + sleep 1 + fi +done <<< "$tags_to_delete_list" + +echo "Cleanup completed successfully" diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/scripts/publish-release-images.sh new file mode 100755 index 000000000..e616604ce --- /dev/null +++ b/.buildkite/scripts/publish-release-images.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# +# Publish release Docker images from ECR to DockerHub. +# Pulls per-arch images, tags with `latest` and versioned tags, pushes them, +# then creates and pushes a multi-arch manifest. + +set -euo pipefail + +RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//') +if [ -z "${RELEASE_VERSION}" ]; then + echo "ERROR: release-version metadata not set" + exit 1 +fi + +COMMIT="$BUILDKITE_COMMIT" +DOCKERHUB_REPO="vllm/vllm-omni" +ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo" + +echo "========================================" +echo "Publishing release images v${RELEASE_VERSION}" +echo " Commit: ${COMMIT}" +echo "========================================" + +# Login to ECR to pull staging images +aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 + +docker pull "${ECR_REPO}:${COMMIT}-x86_64" +docker pull "${ECR_REPO}:${COMMIT}-aarch64" + +docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64" +docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" +docker push "${DOCKERHUB_REPO}:latest-x86_64" +docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" + +docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64" +docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" +docker push "${DOCKERHUB_REPO}:latest-aarch64" +docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" + +docker manifest rm "${DOCKERHUB_REPO}:latest" || true +docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true +docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64" +docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" +docker manifest push "${DOCKERHUB_REPO}:latest" +docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" + +echo "" +echo "Successfully published release images for v${RELEASE_VERSION}" diff --git a/.buildkite/scripts/push-nightly-builds.sh b/.buildkite/scripts/push-nightly-builds.sh new file mode 100755 index 000000000..6dbaf3e4b --- /dev/null +++ b/.buildkite/scripts/push-nightly-builds.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -ex + +# Pull per-arch images from ECR (built earlier in the pipeline) and re-publish +# them to Docker Hub under a nightly tag, then create and push a multi-arch +# manifest pointing at both per-arch tags. + +ORIG_TAG_NAME="$BUILDKITE_COMMIT" +TAG_NAME="nightly" +DOCKERHUB_REPO="vllm/vllm-omni" +ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo" + +echo "Pushing original tag $ORIG_TAG_NAME to new nightly tag name: $TAG_NAME" + +# pull original arch-dependent images from AWS ECR Public +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 +docker pull "$ECR_REPO:$ORIG_TAG_NAME-x86_64" +docker pull "$ECR_REPO:$ORIG_TAG_NAME-aarch64" +# tag arch-dependent images +docker tag "$ECR_REPO:$ORIG_TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-x86_64" +docker tag "$ECR_REPO:$ORIG_TAG_NAME-aarch64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64" +# push arch-dependent images to DockerHub +docker push "$DOCKERHUB_REPO:$TAG_NAME-x86_64" +docker push "$DOCKERHUB_REPO:$TAG_NAME-aarch64" +# push arch-independent manifest to DockerHub +docker manifest create "$DOCKERHUB_REPO:$TAG_NAME" "$DOCKERHUB_REPO:$TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64" --amend +docker manifest create "$DOCKERHUB_REPO:$TAG_NAME-$BUILDKITE_COMMIT" "$DOCKERHUB_REPO:$TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64" --amend +docker manifest push "$DOCKERHUB_REPO:$TAG_NAME" +docker manifest push "$DOCKERHUB_REPO:$TAG_NAME-$BUILDKITE_COMMIT"