Refactor: unify release pipeline with NIGHTLY=1 option (mirror vllm)

- Rename existing CI .buildkite/release-pipeline.yaml -> rebase-pipeline.yaml
  (it builds the CI image and uploads premerge child pipelines, not a release)
- Promote .buildkite/nightly-release-pipeline.yaml -> release-pipeline.yaml,
  refactored to mirror vllm/vllm's release-pipeline.yaml structure:
    - release-version input field
    - block step gating release Docker image builds (NIGHTLY != "1")
    - per-arch x86_64 / aarch64 docker builds pushing to ECR staging
    - multi-arch manifest in ECR
    - nightly DockerHub publish (NIGHTLY == "1") via push-nightly-builds.sh
    - block-gated release DockerHub publish (NIGHTLY != "1") via
      publish-release-images.sh, tagging vX.Y.Z + latest
- Add scripts adapted from vllm:
    push-nightly-builds.sh, cleanup-nightly-builds.sh,
    publish-release-images.sh — all targeting Docker Hub repo vllm/vllm-omni

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
khluuandClaude Opus 4.7 committed 2026-05-07 17:19:55 -07:00
1 parent afbb8af51a
commit 4f12f0c733
6 files changed
+371 -136

No files matched your search

-104
View File
@@ -1,104 +0,0 @@
steps:
- label: "Build and upload wheel"
key: "build-wheel"
agents:
queue: cpu_queue_release
commands:
- "curl -LsSf https://astral.sh/uv/install.sh | sh"
- 'export PATH="$HOME/.local/bin:$PATH"'
- "uv venv --python=3.12 && source .venv/bin/activate"
- "uv pip install --upgrade build"
- "python3 -m build"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
- label: "Generate and upload wheel indices"
depends_on: "build-wheel"
allow_dependency_failure: true
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
- group: "Build release Docker images"
key: "build-release-images"
steps:
- label: "Build release image - x86_64"
key: build-release-image-x86
depends_on: ~
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
--tag vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m) \
--tag vllm/vllm-omni:nightly-$(uname -m) \
--progress plain \
-f docker/Dockerfile.cuda .
# push to ECR (per-arch, commit-pinned) for staging
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
# push to DockerHub (per-arch) so the multi-arch manifest can reference them
- "docker push vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m)"
- "docker push vllm/vllm-omni:nightly-$(uname -m)"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- label: "Build release image - aarch64"
key: build-release-image-arm64
depends_on: ~
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
--tag vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m) \
--tag vllm/vllm-omni:nightly-$(uname -m) \
--progress plain \
-f docker/Dockerfile.cuda .
# push to ECR (per-arch, commit-pinned) for staging
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
# push to DockerHub (per-arch) so the multi-arch manifest can reference them
- "docker push vllm/vllm-omni:$BUILDKITE_COMMIT-$(uname -m)"
- "docker push vllm/vllm-omni:nightly-$(uname -m)"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- label: "Publish multi-arch manifest to DockerHub"
key: publish-multi-arch-dockerhub
depends_on:
- build-release-image-x86
- build-release-image-arm64
agents:
queue: small_cpu_queue_release
commands:
# ECR multi-arch manifest (commit-pinned, useful for downstream tests)
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
# DockerHub multi-arch manifests (nightly + commit-pinned)
- "docker manifest create vllm/vllm-omni:nightly vllm/vllm-omni:nightly-x86_64 vllm/vllm-omni:nightly-aarch64 --amend"
- "docker manifest create vllm/vllm-omni:nightly-$BUILDKITE_COMMIT vllm/vllm-omni:nightly-x86_64 vllm/vllm-omni:nightly-aarch64 --amend"
- "docker manifest push vllm/vllm-omni:nightly"
- "docker manifest push vllm/vllm-omni:nightly-$BUILDKITE_COMMIT"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
+38
View File
@@ -0,0 +1,38 @@
steps:
- label: ":docker: Build image"
key: image-build
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
- "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
- "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
agents:
queue: "cpu_queue_premerge"
# L2 Test
- label: "Upload Ready Pipeline"
depends_on: image-build
key: upload-ready-pipeline
commands:
- buildkite-agent pipeline upload .buildkite/test-ready.yml
agents:
queue: "cpu_queue_premerge"
# L3 Test
- label: "Upload Merge Pipeline"
depends_on: image-build
key: upload-merge-pipeline
commands:
- buildkite-agent pipeline upload .buildkite/test-merge.yml
agents:
queue: "cpu_queue_premerge"
# L4 Test
- label: "Upload Nightly Pipeline"
depends_on: image-build
key: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
- buildkite-agent pipeline upload .buildkite/test-nightly.yml
agents:
queue: "cpu_queue_premerge"
+124 -32
View File
@@ -1,38 +1,130 @@
steps:
- label: ":docker: Build image"
key: image-build
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
- "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
- "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
agents:
queue: "cpu_queue_premerge"
- input: "Provide Release version here"
id: input-release-version
fields:
- text: "What is the release version?"
key: release-version
# L2 Test
- label: "Upload Ready Pipeline"
depends_on: image-build
key: upload-ready-pipeline
commands:
- buildkite-agent pipeline upload .buildkite/test-ready.yml
- label: "Build and upload wheel"
depends_on: ~
key: "build-wheel"
agents:
queue: "cpu_queue_premerge"
queue: cpu_queue_release
commands:
- "curl -LsSf https://astral.sh/uv/install.sh | sh"
- 'export PATH="$HOME/.local/bin:$PATH"'
- "uv venv --python=3.12 && source .venv/bin/activate"
- "uv pip install --upgrade build"
- "python3 -m build"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
# L3 Test
- label: "Upload Merge Pipeline"
depends_on: image-build
key: upload-merge-pipeline
commands:
- buildkite-agent pipeline upload .buildkite/test-merge.yml
- label: "Generate and upload wheel indices"
depends_on: "build-wheel"
allow_dependency_failure: true
agents:
queue: "cpu_queue_premerge"
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
# L4 Test
- label: "Upload Nightly Pipeline"
depends_on: image-build
key: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
- buildkite-agent pipeline upload .buildkite/test-nightly.yml
agents:
queue: "cpu_queue_premerge"
- block: "Unblock to build release Docker images"
depends_on: ~
key: block-build-release-images
if: build.env("NIGHTLY") != "1"
- group: "Build release Docker images"
key: "build-release-images"
depends_on: block-build-release-images
allow_dependency_failure: true
steps:
- label: "Build release image - x86_64"
key: build-release-image-x86
depends_on: ~
agents:
queue: cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
--progress plain \
-f docker/Dockerfile.cuda .
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
# re-tag to default image tag and push, just in case arm64 build fails
- "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
env:
DOCKER_BUILDKIT: "1"
- label: "Build release image - aarch64"
key: build-release-image-arm64
depends_on: ~
agents:
queue: arm64_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- |
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
--progress plain \
-f docker/Dockerfile.cuda .
- "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
env:
DOCKER_BUILDKIT: "1"
- group: "Publish release images"
key: "publish-release-images"
steps:
- label: "Create multi-arch manifest"
depends_on:
- build-release-image-x86
- build-release-image-arm64
id: create-multi-arch-manifest
agents:
queue: small_cpu_queue_release
commands:
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
- label: "Publish nightly multi-arch image to DockerHub"
depends_on:
- create-multi-arch-manifest
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/push-nightly-builds.sh"
# Clean up old nightly builds (keep only last 14)
- "bash .buildkite/scripts/cleanup-nightly-builds.sh"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- block: "Publish release images to DockerHub"
key: block-publish-release-images
depends_on:
- create-multi-arch-manifest
- input-release-version
if: build.env("NIGHTLY") != "1"
- label: "Publish release images to DockerHub"
depends_on:
- block-publish-release-images
key: publish-release-images-dockerhub
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/publish-release-images.sh"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
+129
View File
@@ -0,0 +1,129 @@
#!/bin/bash
set -ex
# Clean up old nightly builds from DockerHub, keeping only the last 14 builds.
# Uses the DockerHub API to list and delete old tags with the given prefix.
# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
# Example: cleanup-nightly-builds.sh "nightly-"
# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-omni"
# Get tag prefix and repo from arguments
TAG_PREFIX="${1:-nightly-}"
REPO="${2:-vllm/vllm-omni}"
echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
# DockerHub API endpoint for the repository
REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
# Get DockerHub credentials from environment
if [ -z "$DOCKERHUB_TOKEN" ]; then
echo "Error: DOCKERHUB_TOKEN environment variable is not set"
exit 1
fi
if [ -z "$DOCKERHUB_USERNAME" ]; then
echo "Error: DOCKERHUB_USERNAME environment variable is not set"
exit 1
fi
# Get DockerHub bearer token
echo "Getting DockerHub bearer token..."
set +x
BEARER_TOKEN=$(curl -s -X POST \
-H "Content-Type: application/json" \
-d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \
"https://hub.docker.com/v2/users/login" | jq -r '.token')
set -x
if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then
echo "Error: Failed to get DockerHub bearer token"
exit 1
fi
# Function to get all tags from DockerHub
get_all_tags() {
local page=1
local all_tags=""
while true; do
set +x
local response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \
"$REPO_API_URL?page=$page&page_size=100")
set -x
# Get both last_updated timestamp and tag name, separated by |
local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
if [ -z "$tags" ]; then
break
fi
all_tags="$all_tags$tags"$'\n'
page=$((page + 1))
done
# Sort by timestamp (newest first) and extract just the tag names
echo "$all_tags" | sort -r | cut -d'|' -f2
}
delete_tag() {
local tag_name="$1"
echo "Deleting tag: $tag_name"
local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
set +x
local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
set -x
if echo "$response" | jq -e '.detail' > /dev/null 2>&1; then
echo "Warning: Failed to delete tag $tag_name: $(echo "$response" | jq -r '.detail')"
else
echo "Successfully deleted tag: $tag_name"
fi
}
# Get all nightly- prefixed tags, sorted by last_updated timestamp (newest first)
echo "Fetching all tags from DockerHub..."
all_tags=$(get_all_tags)
if [ -z "$all_tags" ]; then
echo "No tags found to clean up"
exit 0
fi
# Count total tags
total_tags=$(echo "$all_tags" | wc -l)
echo "Found $total_tags tags"
# Keep only the last 14 builds (including the current one)
tags_to_keep=14
tags_to_delete=$((total_tags - tags_to_keep))
if [ $tags_to_delete -le 0 ]; then
echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)"
exit 0
fi
echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep"
# Get tags to delete (skip the first $tags_to_keep tags)
tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1)))
if [ -z "$tags_to_delete_list" ]; then
echo "No tags to delete"
exit 0
fi
# Delete old tags
echo "Deleting old tags..."
while IFS= read -r tag; do
if [ -n "$tag" ]; then
delete_tag "$tag"
# Add a small delay to avoid rate limiting
sleep 1
fi
done <<< "$tags_to_delete_list"
echo "Cleanup completed successfully"
+50
View File
@@ -0,0 +1,50 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
#
# Publish release Docker images from ECR to DockerHub.
# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
# then creates and pushes a multi-arch manifest.
set -euo pipefail
RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
if [ -z "${RELEASE_VERSION}" ]; then
echo "ERROR: release-version metadata not set"
exit 1
fi
COMMIT="$BUILDKITE_COMMIT"
DOCKERHUB_REPO="vllm/vllm-omni"
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
echo "========================================"
echo "Publishing release images v${RELEASE_VERSION}"
echo " Commit: ${COMMIT}"
echo "========================================"
# Login to ECR to pull staging images
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull "${ECR_REPO}:${COMMIT}-x86_64"
docker pull "${ECR_REPO}:${COMMIT}-aarch64"
docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
docker push "${DOCKERHUB_REPO}:latest-x86_64"
docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
docker push "${DOCKERHUB_REPO}:latest-aarch64"
docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
docker manifest rm "${DOCKERHUB_REPO}:latest" || true
docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
docker manifest push "${DOCKERHUB_REPO}:latest"
docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
echo ""
echo "Successfully published release images for v${RELEASE_VERSION}"
+30
View File
@@ -0,0 +1,30 @@
#!/bin/bash
set -ex
# Pull per-arch images from ECR (built earlier in the pipeline) and re-publish
# them to Docker Hub under a nightly tag, then create and push a multi-arch
# manifest pointing at both per-arch tags.
ORIG_TAG_NAME="$BUILDKITE_COMMIT"
TAG_NAME="nightly"
DOCKERHUB_REPO="vllm/vllm-omni"
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
echo "Pushing original tag $ORIG_TAG_NAME to new nightly tag name: $TAG_NAME"
# pull original arch-dependent images from AWS ECR Public
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull "$ECR_REPO:$ORIG_TAG_NAME-x86_64"
docker pull "$ECR_REPO:$ORIG_TAG_NAME-aarch64"
# tag arch-dependent images
docker tag "$ECR_REPO:$ORIG_TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-x86_64"
docker tag "$ECR_REPO:$ORIG_TAG_NAME-aarch64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64"
# push arch-dependent images to DockerHub
docker push "$DOCKERHUB_REPO:$TAG_NAME-x86_64"
docker push "$DOCKERHUB_REPO:$TAG_NAME-aarch64"
# push arch-independent manifest to DockerHub
docker manifest create "$DOCKERHUB_REPO:$TAG_NAME" "$DOCKERHUB_REPO:$TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64" --amend
docker manifest create "$DOCKERHUB_REPO:$TAG_NAME-$BUILDKITE_COMMIT" "$DOCKERHUB_REPO:$TAG_NAME-x86_64" "$DOCKERHUB_REPO:$TAG_NAME-aarch64" --amend
docker manifest push "$DOCKERHUB_REPO:$TAG_NAME"
docker manifest push "$DOCKERHUB_REPO:$TAG_NAME-$BUILDKITE_COMMIT"