diff --git a/.buildkite/nightly-release-pipeline.yaml b/.buildkite/nightly-release-pipeline.yaml new file mode 100644 index 000000000..373e746de --- /dev/null +++ b/.buildkite/nightly-release-pipeline.yaml @@ -0,0 +1,17 @@ +steps: + - label: "Build and upload wheel" + key: "build-wheel" + agents: + queue: cpu_queue_release + commands: + - "uv pip install --upgrade build" + - "python3 -m build" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + + - label: "Generate and upload wheel indices" + depends_on: "build-wheel" + allow_dependency_failure: true + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/scripts/generate-and-upload-nightly-index.sh new file mode 100755 index 000000000..fc8ddebd7 --- /dev/null +++ b/.buildkite/scripts/generate-and-upload-nightly-index.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -ex + +# Generate and upload wheel indices for all vllm-omni wheels in the commit directory. +# This script should run once after all wheels have been built and uploaded. +# All paths are under the omni/ prefix in the vllm-wheels S3 bucket. + +# ======== setup ======== + +BUCKET="vllm-wheels" +INDICES_OUTPUT_DIR="indices" +PYTHON="${PYTHON_PROG:-python3}" +SUBPATH="omni/$BUILDKITE_COMMIT" +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" + +# detect if python3.12+ is available +has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12) else 0)") +if [[ "$has_new_python" -eq 0 ]]; then + # use new python from docker + docker pull python:3-slim + PYTHON="docker run --rm -v $(pwd):/app -w /app python:3-slim python3" +fi + +echo "Using python interpreter: $PYTHON" +echo "Python version: $($PYTHON --version)" + +# ======== generate and upload indices ======== + +# list all wheels in the commit directory +echo "Existing wheels on S3:" +aws s3 ls "$S3_COMMIT_PREFIX" +obj_json="objects.json" +aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json" +mkdir -p "$INDICES_OUTPUT_DIR" + +# HACK: we do not need regex module here, but it is required by pre-commit hook +# To avoid any external dependency, we simply replace it back to the stdlib re module +sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py + +# Generate indices -- the version is just the commit hash (not omni/{commit}) +# because relative paths are computed between the index and wheel directories, +# both of which live under the omni/ prefix in S3. +$PYTHON .buildkite/scripts/generate-nightly-index.py \ + --version "$BUILDKITE_COMMIT" \ + --current-objects "$obj_json" \ + --output-dir "$INDICES_OUTPUT_DIR" \ + --comment "commit $BUILDKITE_COMMIT" + +# copy indices to /omni/{commit}/ unconditionally +echo "Uploading indices to $S3_COMMIT_PREFIX" +aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX" + +# copy to /omni/nightly/ only if it is on the main branch and not a PR +if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then + echo "Uploading indices to overwrite /omni/nightly/" + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/nightly/" +fi + +# detect version from any wheel in the commit directory +first_wheel_key=$($PYTHON -c "import json; obj=json.load(open('$obj_json')); print(next((c['Key'] for c in obj.get('Contents', []) if c['Key'].endswith('.whl')), ''))") +if [[ -z "$first_wheel_key" ]]; then + echo "Error: No wheels found in $S3_COMMIT_PREFIX" + exit 1 +fi +first_wheel=$(basename "$first_wheel_key") +aws s3 cp "s3://$BUCKET/${first_wheel_key}" "/tmp/${first_wheel}" +version=$(unzip -p "/tmp/${first_wheel}" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2) +rm -f "/tmp/${first_wheel}" +echo "Version in wheel: $version" +pure_version="${version%%+*}" +echo "Pure version (without variant): $pure_version" + +# re-generate and copy to /omni/{version}/ only if it does not have "dev" in the version +if [[ "$version" != *"dev"* ]]; then + echo "Re-generating indices for /omni/$pure_version/" + rm -rf "${INDICES_OUTPUT_DIR:?}" + mkdir -p "$INDICES_OUTPUT_DIR" + # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path + $PYTHON .buildkite/scripts/generate-nightly-index.py \ + --version "$pure_version" \ + --wheel-dir "$BUILDKITE_COMMIT" \ + --current-objects "$obj_json" \ + --output-dir "$INDICES_OUTPUT_DIR" \ + --comment "version $pure_version" + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/$pure_version/" +fi diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py new file mode 100755 index 000000000..d0eeafd71 --- /dev/null +++ b/.buildkite/scripts/generate-nightly-index.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# do not complain about line length (for docstring) +# ruff: noqa: E501 + +import argparse +import json +import sys +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote + +import regex as re + + +def normalize_package_name(name: str) -> str: + """ + Normalize package name according to PEP 503. + https://peps.python.org/pep-0503/#normalized-names + + Replace runs of underscores, hyphens, and periods with a single hyphen, + and lowercase the result. + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +if not sys.version_info >= (3, 12): + raise RuntimeError("This script requires Python 3.12 or higher.") + +INDEX_HTML_TEMPLATE = """ + + + + +{items} + + +""" + + +@dataclass +class WheelFileInfo: + package_name: str + version: str + build_tag: str | None + python_tag: str + abi_tag: str + platform_tag: str + variant: str | None + filename: str + + +def parse_from_filename(file: str) -> WheelFileInfo: + """ + Parse wheel file name to extract metadata. + + The format of wheel names: + {package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl + All versions could contain a variant like '+cu129' or '.cpu' or `.rocm` (or not). + Example: + vllm_omni-0.1.0-py3-none-any.whl + vllm_omni-0.1.0.dev5+g1234567-py3-none-any.whl + vllm_omni-0.1.0.dev5+g1234567.cpu-py3-none-any.whl + vllm_omni-0.1.0.dev5+g1234567.rocm-py3-none-any.whl + """ + wheel_file_re = re.compile( + r"^(?P.+)-(?P[^-]+?)(-(?P[^-]+))?-(?P[^-]+)-(?P[^-]+)-(?P[^-]+)\.whl$" + ) + match = wheel_file_re.match(file) + if not match: + raise ValueError(f"Invalid wheel file name: {file}") + + package_name = match.group("package_name") + version = match.group("version") + build_tag = match.group("build_tag") + python_tag = match.group("python_tag") + abi_tag = match.group("abi_tag") + platform_tag = match.group("platform_tag") + + # extract variant from version + variant = None + if "dev" in version: + ver_after_dev = version.split("dev")[-1] + if "." in ver_after_dev: + variant = ver_after_dev.split(".")[-1] + version = version.removesuffix("." + variant) + else: + if "+" in version: + version_part, suffix = version.split("+", 1) + # Only treat known patterns as variants (rocmXXX, cuXXX, cpu) + # Git hashes and other suffixes are NOT variants + if suffix.startswith(("rocm", "cu", "cpu")): + variant = suffix + version = version_part + # Otherwise keep the full version string (variant stays None) + + return WheelFileInfo( + package_name=package_name, + version=version, + build_tag=build_tag, + python_tag=python_tag, + abi_tag=abi_tag, + platform_tag=platform_tag, + variant=variant, + filename=file, + ) + + +def generate_project_list(subdir_names: list[str], comment: str = "") -> str: + """ + Generate project list HTML content linking to each project & variant subdirectory. + """ + href_tags = [] + for name in sorted(subdir_names): + name = name.strip("/").strip(".") + href_tags.append(f' {name}/
') + return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) + + +def generate_package_index_and_metadata( + wheel_files: list[WheelFileInfo], + wheel_base_dir: Path, + index_base_dir: Path, + comment: str = "", +) -> tuple[str, str]: + """ + Generate package index HTML content for a specific package, linking to actual wheel files. + """ + href_tags = [] + metadata = [] + for file in sorted(wheel_files, key=lambda x: x.filename): + relative_path = ( + wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename + ) + # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B' + # NOTE: this is AWS S3 specific behavior! + file_path_quoted = quote(relative_path.as_posix(), safe=":%/") + href_tags.append(f' {file.filename}
') + file_meta = asdict(file) + file_meta["path"] = file_path_quoted + metadata.append(file_meta) + index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) + metadata_str = json.dumps(metadata, indent=2) + return index_str, metadata_str + + +def generate_index_and_metadata( + whl_files: list[str], + wheel_base_dir: Path, + index_base_dir: Path, + default_variant: str | None = None, + alias_to_default: str | None = None, + comment: str = "", +): + """ + Generate index for all wheel files. + + Args: + whl_files (list[str]): List of wheel files (must be directly under `wheel_base_dir`). + wheel_base_dir (Path): Base directory for wheel files. + index_base_dir (Path): Base directory to store index files. + default_variant (str | None): The default variant name, if any. + alias_to_default (str | None): Alias variant name for the default variant, if any. + comment (str | None): Optional comment to include in the generated HTML files. + + Index directory structure: + index_base_dir/ (hosted at wheels.vllm.ai/omni/{nightly,$commit,$version}/) + index.html # project list + vllm-omni/ + index.html # package index, pointing to actual files in wheel_base_dir (relative path) + metadata.json # machine-readable metadata + cpu/ # cpu variant subdirectory + index.html + vllm-omni/ + index.html + metadata.json + ... + """ + + parsed_files = [parse_from_filename(f) for f in whl_files] + + if not parsed_files: + print("No wheel files found, skipping index generation.") + return + + # For ROCm builds: inherit variant from vllm-omni wheel + rocm_variant = None + for file in parsed_files: + if ( + file.package_name == "vllm_omni" + and file.variant + and file.variant.startswith("rocm") + ): + rocm_variant = file.variant + print(f"Detected ROCm variant from vllm-omni: {rocm_variant}") + break + + # Apply ROCm variant to all wheels without a variant + if rocm_variant: + for file in parsed_files: + if file.variant is None: + file.variant = rocm_variant + print(f"Inherited variant '{rocm_variant}' for {file.filename}") + + # Group by variant + variant_to_files: dict[str, list[WheelFileInfo]] = {} + for file in parsed_files: + variant = file.variant or "default" + if variant not in variant_to_files: + variant_to_files[variant] = [] + variant_to_files[variant].append(file) + + print(f"Found variants: {list(variant_to_files.keys())}") + + # sanity check for default variant + if default_variant: + if "default" in variant_to_files: + raise ValueError( + "All wheel files must have variant suffixes when `default_variant` is specified." + ) + if default_variant not in variant_to_files: + raise ValueError( + f"Default variant '{default_variant}' not found among wheel files." + ) + + if alias_to_default: + if "default" not in variant_to_files: + print( + "[WARN] Alias to default variant specified, but no default variant found." + ) + elif alias_to_default in variant_to_files: + raise ValueError( + f"Alias variant name '{alias_to_default}' already exists among wheel files." + ) + else: + variant_to_files[alias_to_default] = variant_to_files["default"].copy() + print(f"Alias variant '{alias_to_default}' created for default variant.") + + # Generate comment in HTML header + comment_str = f" ({comment})" if comment else "" + comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}" + + # Generate index for each variant + subdir_names = set() + for variant, files in variant_to_files.items(): + if variant == "default": + variant_dir = index_base_dir + else: + variant_dir = index_base_dir / variant + subdir_names.add(variant) + + variant_dir.mkdir(parents=True, exist_ok=True) + + # gather all package names in this variant (normalized per PEP 503) + packages = set(normalize_package_name(f.package_name) for f in files) + if variant == "default": + subdir_names = subdir_names.union(packages) + else: + project_list_str = generate_project_list(sorted(packages), comment_tmpl) + with open(variant_dir / "index.html", "w") as f: + f.write(project_list_str) + + for package in packages: + package_files = [ + f for f in files if normalize_package_name(f.package_name) == package + ] + package_dir = variant_dir / package + package_dir.mkdir(parents=True, exist_ok=True) + index_str, metadata_str = generate_package_index_and_metadata( + package_files, wheel_base_dir, package_dir, comment + ) + with open(package_dir / "index.html", "w") as f: + f.write(index_str) + with open(package_dir / "metadata.json", "w") as f: + f.write(metadata_str) + + # Generate top-level project list index + project_list_str = generate_project_list(sorted(subdir_names), comment_tmpl) + with open(index_base_dir / "index.html", "w") as f: + f.write(project_list_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process nightly build wheel files to generate indices." + ) + parser.add_argument( + "--version", + type=str, + required=True, + help="Version string for the current build (e.g., commit hash)", + ) + parser.add_argument( + "--current-objects", + type=str, + required=True, + help="Path to JSON file containing current S3 objects listing in this version directory", + ) + parser.add_argument( + "--output-dir", + type=str, + required=True, + help="Directory to store generated index files", + ) + parser.add_argument( + "--wheel-dir", + type=str, + default=None, + help="Directory containing wheel files (default to be same as `version`)", + ) + parser.add_argument( + "--alias-to-default", + type=str, + default=None, + help="Alias variant name for the default variant", + ) + parser.add_argument( + "--comment", + type=str, + default="", + help="Optional comment string to include in generated HTML files", + ) + + args = parser.parse_args() + + version = args.version + if "\\" in version: + raise ValueError("Version string must not contain backslashes.") + if "/" in version: + raise ValueError("Version string must not contain slashes.") + current_objects_path = Path(args.current_objects) + output_dir = Path(args.output_dir) + if not output_dir.exists(): + output_dir.mkdir(parents=True, exist_ok=True) + + # Read current objects JSON + with open(current_objects_path) as f: + current_objects: dict[str, list[dict[str, Any]]] = json.load(f) + + # Extract wheel file keys + wheel_files = [] + for item in current_objects.get("Contents", []): + key: str = item["Key"] + if key.endswith(".whl"): + wheel_files.append(key.split("/")[-1]) # only the filename is used + + print(f"Found {len(wheel_files)} wheel files for version {version}: {wheel_files}") + + # keep only "official" files for a non-nightly version (specified by cli args) + PY_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+([a-zA-Z0-9.+-]*)?$") + if PY_VERSION_RE.match(version): + wheel_files = list( + filter(lambda x: version in x and "dev" not in x, wheel_files) + ) + print(f"Non-nightly version detected, wheel files used: {wheel_files}") + else: + print("Nightly version detected, keeping all wheel files.") + + # Generate index and metadata + # Wheels are at s3://vllm-wheels/omni/{wheel_dir}/ + # Indices are at s3://vllm-wheels/omni// + # Relative paths work because both are under the omni/ prefix. + if args.wheel_dir: + wheel_dir = args.wheel_dir.strip().rstrip("/") + else: + wheel_dir = version + wheel_base_dir = Path(output_dir).parent / wheel_dir + index_base_dir = Path(output_dir) + + generate_index_and_metadata( + whl_files=wheel_files, + wheel_base_dir=wheel_base_dir, + index_base_dir=index_base_dir, + default_variant=None, + alias_to_default=args.alias_to_default, + comment=args.comment.strip(), + ) + print(f"Successfully generated index and metadata in {output_dir}") diff --git a/.buildkite/scripts/upload-nightly-wheels.sh b/.buildkite/scripts/upload-nightly-wheels.sh new file mode 100755 index 000000000..d50da1ded --- /dev/null +++ b/.buildkite/scripts/upload-nightly-wheels.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -ex + +# Upload a single wheel to S3 under the omni/ prefix. +# Index generation is handled separately by generate-and-upload-nightly-index.sh. + +BUCKET="vllm-wheels" +SUBPATH="omni/$BUILDKITE_COMMIT" +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" + +# ========= collect & upload the wheel ========== + +# python3 -m build outputs to dist/ by default +wheel_files=(dist/*.whl) + +# Check that exactly one wheel is found +if [[ ${#wheel_files[@]} -ne 1 ]]; then + echo "Error: Expected exactly one wheel file in dist/, but found ${#wheel_files[@]}" + exit 1 +fi +wheel="${wheel_files[0]}" + +echo "Uploading wheel: $wheel" + +# Extract the version from the wheel +version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2) +echo "Version in wheel: $version" + +# Upload wheel to S3 +aws s3 cp "$wheel" "$S3_COMMIT_PREFIX" + +echo "Wheel uploaded to $S3_COMMIT_PREFIX. Index generation is handled by a separate step."