[CI/Perf] Optimize Docker Caching (#5133)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
Signed-off-by: Doug Smith <dougbtv@gmail.com>
Co-authored-by: dougbtv <dosmith@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

.buildkite/cuda/pipeline.yml

@@ -19,10 +19,7 @@ steps:
     key: image-build
     if: __IMAGE_BUILD_IF__
     commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
     agents:
       queue: "cpu_queue_premerge"
 

.buildkite/cuda/rebase-pipeline.yml

@@ -2,10 +2,7 @@ steps:
   - label: ":docker: Build image"
     key: image-build
     commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
     agents:
       queue: "cpu_queue_premerge"
 

.buildkite/cuda/scripts/build-ci-image.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build and push the CI Docker image with registry-based layer caching.
+#
+# Usage: build-ci-image.sh
+#
+# Expects the following environment variables (set by Buildkite):
+#   BUILDKITE_COMMIT          - commit SHA used as the image tag
+ECR_NAMESPACE="public.ecr.aws/q9t5s3a7"
+REGISTRY="${ECR_NAMESPACE}/vllm-ci-test-repo"
+REGION="us-east-1"
+DOCKERFILE="docker/Dockerfile.ci"
+BUILDER_NAME="vllm-omni-builder"
+
+# Ensure that the env vars are actually set, otherwise exit early
+if [ -z "${BUILDKITE_COMMIT:-}" ]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set"
+    exit 1
+fi
+echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
+
+# Authenticate to ECR Public
+aws ecr-public get-login-password --region "$REGION" \
+    | docker login --username AWS --password-stdin "$ECR_NAMESPACE"
+
+# Compute the cache tag for the dependencies image as the hash of the contents.
+# This is because dependencies don't change often, so most PRs will have the same hash.
+#
+# The Dockerfile is also included to avoid churn in the cached image based on the vLLM version.
+DEP_FILES="pyproject.toml setup.py requirements/*.txt docker/Dockerfile.ci"
+CACHE_KEY=$(cat ${DEP_FILES} | sha256sum | cut -c1-16)
+CACHE_TAG="deps-cache-${CACHE_KEY}"
+echo "Cache key: ${CACHE_TAG}"
+
+# Set up buildx with docker-container driver; we need to do this
+# since cache export is not supported for the default docker driver
+# that is running in the CI.
+docker buildx inspect $BUILDER_NAME >/dev/null 2>&1 \
+|| docker buildx create --name $BUILDER_NAME --driver docker-container
+docker buildx use $BUILDER_NAME
+
+echo "Building image tag: ${REGISTRY}:${BUILDKITE_COMMIT}"
+
+# Build + push the image to the registry, using the hash from the dependencies.
+# Note that the caching here is layerwise, so we're just making sure we always
+# can cache hit on everything but the last copy at the moment.
+docker buildx build --push --progress=plain \
+    --cache-from "type=registry,ref=${REGISTRY}:${CACHE_TAG}" \
+    --cache-to "type=registry,ref=${REGISTRY}:${CACHE_TAG},mode=max,compression=zstd" \
+    --file "$DOCKERFILE" \
+    -t "${REGISTRY}:${BUILDKITE_COMMIT}" .

.gitignore

@@ -259,7 +259,6 @@ configs/production.yaml
 configs/development.yaml
 
 # Docker
-.dockerignore
 Dockerfile.dev
 discussion
 tmp_test

docker/Dockerfile.ci

@@ -3,10 +3,22 @@
 # be updated on every rebase to match the new upstream vllm commit/tag.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
 ARG VLLM_BASE_TAG=v0.25.0
-FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
+
+# We use registry-based layer caching for the dependencies,
+# where the tag is determined based on the hash of the contents
+# of files like pyproject.toml, etc. The deps stage should have
+# everything except the actual project dir, since that should not
+# change much across branches.
+FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} AS deps
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
-COPY . .
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
+
+# Create a vllm_omni dir so that we can install the python package
+# without failing to write the version files; we'll copy over it
+# after installing
+RUN mkdir -p vllm_omni
 
 # Install system dependencies
 RUN apt-get update && \
@@ -14,9 +26,33 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Copy dependency related things first and install deps/
+# We set the version str to 0.0.0 since it doesn't matter
+# for CI
+COPY pyproject.toml setup.py ./
+COPY requirements/ requirements/
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
 RUN uv pip install --system ".[dev]"
 
+# The final stage just copies the working dir on top of the
+# dependencies image. This uses --link so that we can skip
+# the image pull in Buildkite when we have a cache hit.
+#
+# IMPORTANT: Do not make any modifications that depend on the
+# file system after the --link copy, otherwise we'd need to pull
+# the heavy base image to build it. This is the reason we copy
+# link & update the pythonpath to point at the copied code instead
+# of running a new pip install on it.
+FROM deps
+ARG APP_DIR=/workspace/vllm-omni
+# Add APP_DIR to the pythonpath, since the entrypoint stub
+# that was installed in the earlier step must point at the copied code.
+#
+# NOTE: this will probably warn about undefined vars, but is expected.
+# See: https://github.com/docker/buildx/issues/2751
+ENV PYTHONPATH=${APP_DIR}${PYTHONPATH:+:${PYTHONPATH}}
+WORKDIR ${APP_DIR}
 
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+COPY --link . .
 
 ENTRYPOINT []

docker/Dockerfile.ci.dockerignore

@@ -0,0 +1,37 @@
+# NOTE: this is largely taken from vLLM
+
+# Exclude .git for CI builds — version is set via SETUPTOOLS_SCM_PRETEND_VERSION
+.git
+.venv
+/build
+dist
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.mypy_cache
+
+# Distribution / packaging
+.Python
+/build/
+cmake-build-*/
+CMakeUserPresets.json
+develop-eggs/
+/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+rust/target/