diff --git a/.buildkite/nightly-release-pipeline.yaml b/.buildkite/nightly-release-pipeline.yaml deleted file mode 100644 index 25c52ba3b..000000000 --- a/.buildkite/nightly-release-pipeline.yaml +++ /dev/null @@ -1,20 +0,0 @@ -steps: - - label: "Build and upload wheel" - key: "build-wheel" - agents: - queue: cpu_queue_release - commands: - - "curl -LsSf https://astral.sh/uv/install.sh | sh" - - 'export PATH="$HOME/.local/bin:$PATH"' - - "uv venv --python=3.12 && source .venv/bin/activate" - - "uv pip install --upgrade build" - - "python3 -m build" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - - label: "Generate and upload wheel indices" - depends_on: "build-wheel" - allow_dependency_failure: true - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/rebase-pipeline.yaml new file mode 100644 index 000000000..edd98fafd --- /dev/null +++ b/.buildkite/rebase-pipeline.yaml @@ -0,0 +1,38 @@ +steps: + - label: ":docker: Build image" + key: image-build + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." + - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" + agents: + queue: "cpu_queue_premerge" + + # L2 Test + - label: "Upload Ready Pipeline" + depends_on: image-build + key: upload-ready-pipeline + commands: + - buildkite-agent pipeline upload .buildkite/test-ready.yml + agents: + queue: "cpu_queue_premerge" + + # L3 Test + - label: "Upload Merge Pipeline" + depends_on: image-build + key: upload-merge-pipeline + commands: + - buildkite-agent pipeline upload .buildkite/test-merge.yml + agents: + queue: "cpu_queue_premerge" + + # L4 Test + - label: "Upload Nightly Pipeline" + depends_on: image-build + key: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" + commands: + - buildkite-agent pipeline upload .buildkite/test-nightly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index edd98fafd..c560fefe1 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -1,38 +1,110 @@ steps: - - label: ":docker: Build image" - key: image-build - commands: - - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ." - - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" - agents: - queue: "cpu_queue_premerge" + - input: "Provide Release version here" + id: input-release-version + fields: + - text: "What is the release version?" + key: release-version - # L2 Test - - label: "Upload Ready Pipeline" - depends_on: image-build - key: upload-ready-pipeline - commands: - - buildkite-agent pipeline upload .buildkite/test-ready.yml + - label: "Build and upload wheel" + depends_on: ~ + key: "build-wheel" agents: - queue: "cpu_queue_premerge" + queue: cpu_queue_release + commands: + - "curl -LsSf https://astral.sh/uv/install.sh | sh" + - 'export PATH="$HOME/.local/bin:$PATH"' + - "uv venv --python=3.12 && source .venv/bin/activate" + - "uv pip install --upgrade build" + - "python3 -m build" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" - # L3 Test - - label: "Upload Merge Pipeline" - depends_on: image-build - key: upload-merge-pipeline - commands: - - buildkite-agent pipeline upload .buildkite/test-merge.yml + - label: "Generate and upload wheel indices" + depends_on: "build-wheel" + allow_dependency_failure: true agents: - queue: "cpu_queue_premerge" + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" - # L4 Test - - label: "Upload Nightly Pipeline" - depends_on: image-build - key: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" - commands: - - buildkite-agent pipeline upload .buildkite/test-nightly.yml + - block: "Upload release wheels to PyPI" + depends_on: "build-wheel" + key: block-upload-release-wheels + + - label: "Upload release wheels to PyPI" + depends_on: + - block-upload-release-wheels + - input-release-version + key: upload-release-wheels agents: - queue: "cpu_queue_premerge" + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/publish-release-wheels-pypi.sh" + + - block: "Unblock to build release Docker images" + depends_on: ~ + key: block-build-release-images + + - group: "Build release Docker images" + key: "build-release-images" + depends_on: block-build-release-images + allow_dependency_failure: true + steps: + - label: "Build release image - x86_64" + key: build-release-image-x86 + depends_on: ~ + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + --build-arg max_jobs=16 \ + --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.cuda . + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + # re-tag to default image tag and push, just in case arm64 build fails + - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT" + env: + DOCKER_BUILDKIT: "1" + + - label: "Build release image - aarch64" + key: build-release-image-arm64 + depends_on: ~ + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + --build-arg max_jobs=16 \ + --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.cuda . + - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + env: + DOCKER_BUILDKIT: "1" + + - block: "Publish release images to DockerHub" + key: block-publish-release-images + depends_on: + - build-release-image-x86 + - build-release-image-arm64 + - input-release-version + + - label: "Publish release images to DockerHub" + depends_on: + - block-publish-release-images + key: publish-release-images-dockerhub + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/publish-release-images.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/scripts/publish-release-images.sh new file mode 100755 index 000000000..e616604ce --- /dev/null +++ b/.buildkite/scripts/publish-release-images.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# +# Publish release Docker images from ECR to DockerHub. +# Pulls per-arch images, tags with `latest` and versioned tags, pushes them, +# then creates and pushes a multi-arch manifest. + +set -euo pipefail + +RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//') +if [ -z "${RELEASE_VERSION}" ]; then + echo "ERROR: release-version metadata not set" + exit 1 +fi + +COMMIT="$BUILDKITE_COMMIT" +DOCKERHUB_REPO="vllm/vllm-omni" +ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo" + +echo "========================================" +echo "Publishing release images v${RELEASE_VERSION}" +echo " Commit: ${COMMIT}" +echo "========================================" + +# Login to ECR to pull staging images +aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 + +docker pull "${ECR_REPO}:${COMMIT}-x86_64" +docker pull "${ECR_REPO}:${COMMIT}-aarch64" + +docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64" +docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" +docker push "${DOCKERHUB_REPO}:latest-x86_64" +docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" + +docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64" +docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" +docker push "${DOCKERHUB_REPO}:latest-aarch64" +docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" + +docker manifest rm "${DOCKERHUB_REPO}:latest" || true +docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true +docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64" +docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64" +docker manifest push "${DOCKERHUB_REPO}:latest" +docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" + +echo "" +echo "Successfully published release images for v${RELEASE_VERSION}" diff --git a/.buildkite/scripts/publish-release-wheels-pypi.sh b/.buildkite/scripts/publish-release-wheels-pypi.sh new file mode 100755 index 000000000..c107e649e --- /dev/null +++ b/.buildkite/scripts/publish-release-wheels-pypi.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -e + +BUCKET="vllm-wheels" +SUBPATH="omni/$BUILDKITE_COMMIT" +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" + +RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//') +if [[ -z "$RELEASE_VERSION" ]]; then + echo "[FATAL] release-version metadata not set." + exit 1 +fi +echo "Release version: $RELEASE_VERSION" + +# if [[ -z "$PYPI_TOKEN" ]]; then +# echo "[FATAL] PYPI_TOKEN is not set." +# exit 1 +# else +# export TWINE_USERNAME="__token__" +# export TWINE_PASSWORD="$PYPI_TOKEN" +# fi + +set -x + +ALL_DIR=/tmp/vllm-omni-release-all +DIST_DIR=/tmp/vllm-omni-release-dist +mkdir -p "$ALL_DIR" "$DIST_DIR" + +echo "========================================" +echo "All wheels in S3 (no filters):" +echo "========================================" +aws s3 cp --recursive "$S3_COMMIT_PREFIX" "$ALL_DIR" +ls -la "$ALL_DIR" + +echo "" +echo "========================================" +echo "Filtered wheels (include: vllm_omni-${RELEASE_VERSION}*.whl, exclude: *dev*):" +echo "========================================" +aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR" +ls -la "$DIST_DIR" + +PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*") +if [[ -z "$PYPI_WHEEL_FILES" ]]; then + echo "[FATAL] No wheels found for version ${RELEASE_VERSION}" + exit 1 +fi + +echo "" +echo "========================================" +echo "Wheels that would be uploaded to PyPI:" +echo "========================================" +echo "$PYPI_WHEEL_FILES" + +# python3 -m twine check $PYPI_WHEEL_FILES +# python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES +# echo "Wheels uploaded to PyPI"