[Refactor][CI] Reorganize .buildkite directory by platform (#5119)
This commit is contained in:
wangyu authored and GitHub committed
1 parent
c23dbfe111
commit
0d8c632688
38 files changed
+114
-438
No files matched your search
@@ -154,14 +154,14 @@ upload_pipeline() {
|
||||
fi
|
||||
|
||||
# Use local template file for vllm-omni
|
||||
cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
|
||||
cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2
|
||||
|
||||
|
||||
# (WIP) Use pipeline generator instead of jinja template
|
||||
if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
|
||||
if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
|
||||
python -m pip install click pydantic
|
||||
python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
|
||||
buildkite-agent pipeline upload .buildkite/pipeline.yaml
|
||||
python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
|
||||
buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
|
||||
exit 0
|
||||
fi
|
||||
echo "List file diff: $LIST_FILE_DIFF"
|
||||
@@ -171,7 +171,7 @@ upload_pipeline() {
|
||||
|
||||
FAIL_FAST=$(fail_fast)
|
||||
|
||||
cd .buildkite
|
||||
cd .buildkite/amd
|
||||
|
||||
# Select test definition file: merge suite for main, ready suite for PRs.
|
||||
# For debugging, DEBUG_TEST_YAML can override the selection — accepts
|
||||
@@ -182,7 +182,7 @@ upload_pipeline() {
|
||||
TEST_YAML="test-amd-merge.yml"
|
||||
;;
|
||||
ready)
|
||||
TEST_YAML="test-amd-ready.yaml"
|
||||
TEST_YAML="test-amd-ready.yml"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
|
||||
@@ -193,7 +193,7 @@ upload_pipeline() {
|
||||
elif [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||
TEST_YAML="test-amd-merge.yml"
|
||||
else
|
||||
TEST_YAML="test-amd-ready.yaml"
|
||||
TEST_YAML="test-amd-ready.yml"
|
||||
fi
|
||||
|
||||
(
|
||||
+1
-1
@@ -56,7 +56,7 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
|
||||
# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
|
||||
# sudo apt-get update && sudo apt-get install -y awscli
|
||||
## Use safe docker login helper to prevent race conditions
|
||||
# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
|
||||
# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh"
|
||||
# safe_docker_login_ecr_public
|
||||
|
||||
## Pull the container from AMD Docker Hub
|
||||
File renamed without changes.
File renamed without changes.
@@ -57,7 +57,7 @@ steps:
|
||||
{% endif %}
|
||||
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
|
||||
{% set indented_cmd = cmd_body | replace("\n", "\n ") %}
|
||||
command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
|
||||
command: bash .buildkite/amd/scripts/run-amd-test.sh
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
{% if step.env %}
|
||||
@@ -104,7 +104,7 @@ steps:
|
||||
{% endif %}
|
||||
{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
|
||||
{% set indented_cmd = cmd_body | replace("\n", "\n ") %}
|
||||
command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
|
||||
command: bash .buildkite/amd/scripts/run-amd-test.sh
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
{% if step.env %}
|
||||
@@ -9,7 +9,7 @@ steps:
|
||||
- label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline"
|
||||
key: upload-ci-pipeline
|
||||
commands:
|
||||
- "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml"
|
||||
- "python3 .buildkite/cuda/scripts/upload_pipeline.py --upload .buildkite/cuda/pipeline.yml"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -37,7 +37,7 @@ steps:
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-ready.yml
|
||||
python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -52,7 +52,7 @@ steps:
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-merge.yml
|
||||
python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -70,7 +70,7 @@ steps:
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
))
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-nightly.yml
|
||||
- buildkite-agent pipeline upload .buildkite/cuda/test-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -82,6 +82,6 @@ steps:
|
||||
(build.branch == "main" && build.env("WEEKLY") == "1") ||
|
||||
(build.branch != "main" && build.pull_request.labels includes "weekly-test")
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-weekly.yml
|
||||
- buildkite-agent pipeline upload .buildkite/cuda/test-weekly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -14,7 +14,7 @@ steps:
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml
|
||||
- python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -23,7 +23,7 @@ steps:
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml
|
||||
- python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
@@ -33,6 +33,6 @@ steps:
|
||||
key: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml
|
||||
- python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
@@ -16,16 +16,16 @@ Usage:
|
||||
python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
|
||||
|
||||
# Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
|
||||
python3 upload_pipeline.py --upload .buildkite/pipeline.yml
|
||||
python3 upload_pipeline.py --upload .buildkite/cuda/pipeline.yml
|
||||
|
||||
# Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
|
||||
python3 upload_pipeline.py --upload .buildkite/test-merge.yml
|
||||
python3 upload_pipeline.py --upload .buildkite/cuda/test-merge.yml
|
||||
|
||||
# Full suite (rebase pipeline): keep all steps, still strip the uploader-only key:
|
||||
python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml
|
||||
python3 upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
|
||||
|
||||
# Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies:
|
||||
python3 upload_pipeline.py --upload --e2e .buildkite/test-ready.yml
|
||||
python3 upload_pipeline.py --upload --e2e .buildkite/cuda/test-merge.yml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -47,7 +47,7 @@ except ImportError:
|
||||
import yaml
|
||||
|
||||
LOG = "upload_pipeline"
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent.parent
|
||||
DOC_SEP = "\n---\n"
|
||||
BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
|
||||
E2E_GROUP_MARKER = "E2E Test"
|
||||
@@ -475,7 +475,7 @@ def render_pipeline(
|
||||
|
||||
# ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still
|
||||
# stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds
|
||||
# run the full e2e suite (see .buildkite/rebase-pipeline.yaml).
|
||||
# run the full e2e suite (see .buildkite/cuda/rebase-pipeline.yml).
|
||||
if force_all or e2e_only:
|
||||
changed_files = None
|
||||
|
||||
@@ -509,8 +509,8 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"pipeline",
|
||||
nargs="?",
|
||||
default=".buildkite/pipeline.yml",
|
||||
help="Pipeline YAML path (default: .buildkite/pipeline.yml)",
|
||||
default=".buildkite/cuda/pipeline.yml",
|
||||
help="Pipeline YAML path (default: .buildkite/cuda/pipeline.yml)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--upload",
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,9 +1,9 @@
|
||||
# .buildkite/pipeline-intel.yaml
|
||||
# .buildkite/intel/pipeline-intel.yml
|
||||
steps:
|
||||
- group: "INTEL Tests"
|
||||
steps:
|
||||
- label: "Intel: XPU Test"
|
||||
command: "bash .buildkite/scripts/hardware_ci/run-xpu-test.sh"
|
||||
command: "bash .buildkite/intel/scripts/run-xpu-test.sh"
|
||||
agents:
|
||||
queue: "intel-gpu-omni"
|
||||
env:
|
||||
+1
-1
@@ -11,7 +11,7 @@ fi
|
||||
|
||||
upload_pipeline() {
|
||||
echo "--- 🛠 Preparing Intel pipeline"
|
||||
buildkite-agent pipeline upload .buildkite/pipeline-intel.yaml
|
||||
buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml
|
||||
}
|
||||
|
||||
get_diff() {
|
||||
File renamed without changes.
@@ -88,7 +88,7 @@
|
||||
# # queue: "ascend-a3"
|
||||
# # resource_class: "npu-2"
|
||||
# # command: |
|
||||
# # buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
|
||||
# # buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
|
||||
|
||||
# # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
# - label: "Upload Nightly Pipeline"
|
||||
@@ -104,7 +104,7 @@
|
||||
# build.pull_request.labels includes "diffusion-x2v-test"
|
||||
# ))
|
||||
# commands:
|
||||
# - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
|
||||
# - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
|
||||
# agents:
|
||||
# queue: "ascend-a3"
|
||||
# resource_class: "npu-2"
|
||||
@@ -171,7 +171,7 @@ steps:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
command: |
|
||||
buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
|
||||
buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
|
||||
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
@@ -189,7 +189,7 @@ steps:
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
))
|
||||
commands:
|
||||
- buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
|
||||
- buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
|
||||
agents:
|
||||
queue: "ascend-a2b3"
|
||||
resource_class: "npu-1"
|
||||
File renamed without changes.
File renamed without changes.
@@ -16,7 +16,7 @@ steps:
|
||||
- "uv venv --python=3.12 && source .venv/bin/activate"
|
||||
- "uv pip install --upgrade build"
|
||||
- "python3 -m build"
|
||||
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
|
||||
- "bash .buildkite/release/scripts/upload-nightly-wheels.sh"
|
||||
|
||||
- label: "Generate and upload wheel indices"
|
||||
depends_on: "build-wheel"
|
||||
@@ -24,7 +24,7 @@ steps:
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
|
||||
- "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh"
|
||||
|
||||
- block: "Upload release wheels to PyPI"
|
||||
depends_on: "build-wheel"
|
||||
@@ -38,7 +38,7 @@ steps:
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
|
||||
- "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
|
||||
|
||||
- block: "Unblock to build release Docker images"
|
||||
depends_on: ~
|
||||
@@ -100,7 +100,7 @@ steps:
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/scripts/publish-release-images.sh"
|
||||
- "bash .buildkite/release/scripts/publish-release-images.sh"
|
||||
plugins:
|
||||
- docker-login#v3.0.0:
|
||||
username: vllmbot
|
||||
+3
-3
@@ -36,12 +36,12 @@ mkdir -p "$INDICES_OUTPUT_DIR"
|
||||
|
||||
# HACK: we do not need regex module here, but it is required by pre-commit hook
|
||||
# To avoid any external dependency, we simply replace it back to the stdlib re module
|
||||
sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak
|
||||
sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak
|
||||
|
||||
# Generate indices -- the version is just the commit hash (not omni/{commit})
|
||||
# because relative paths are computed between the index and wheel directories,
|
||||
# both of which live under the omni/ prefix in S3.
|
||||
$PYTHON .buildkite/scripts/generate-nightly-index.py \
|
||||
$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
|
||||
--version "$BUILDKITE_COMMIT" \
|
||||
--current-objects "$obj_json" \
|
||||
--output-dir "$INDICES_OUTPUT_DIR" \
|
||||
@@ -78,7 +78,7 @@ if [[ "$version" != *"dev"* ]]; then
|
||||
rm -rf "${INDICES_OUTPUT_DIR:?}"
|
||||
mkdir -p "$INDICES_OUTPUT_DIR"
|
||||
# wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
|
||||
$PYTHON .buildkite/scripts/generate-nightly-index.py \
|
||||
$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
|
||||
--version "$s3_version" \
|
||||
--wheel-dir "$BUILDKITE_COMMIT" \
|
||||
--current-objects "$obj_json" \
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,73 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Helper function to safely login to ECR Public with per-job config isolation
|
||||
# Uses DOCKER_CONFIG environment variable to prevent race conditions
|
||||
#
|
||||
# This script prevents the "device or resource busy" error by giving each
|
||||
# Buildkite job its own isolated Docker config directory.
|
||||
#
|
||||
# Usage:
|
||||
# source docker_login_ecr_public.sh && safe_docker_login_ecr_public
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
ECR_REGISTRY="public.ecr.aws"
|
||||
|
||||
setup_isolated_docker_config() {
|
||||
# Use BUILDKITE_JOB_ID for job-specific isolation
|
||||
# Fallback to PID if running outside Buildkite
|
||||
local job_id="${BUILDKITE_JOB_ID:-$$}"
|
||||
|
||||
# Set Docker config to job-specific directory
|
||||
export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
mkdir -p "$DOCKER_CONFIG"
|
||||
|
||||
echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
|
||||
}
|
||||
|
||||
check_docker_auth() {
|
||||
# Check if already authenticated to the given registry
|
||||
# Returns 0 if authenticated, 1 if not
|
||||
local registry="$1"
|
||||
|
||||
# Check if credentials exist in the isolated config
|
||||
if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
|
||||
# Check if registry is present in config
|
||||
if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
safe_docker_login_ecr_public() {
|
||||
# Setup isolated config first
|
||||
setup_isolated_docker_config
|
||||
|
||||
local registry="$ECR_REGISTRY"
|
||||
|
||||
# Check if already authenticated (within this job)
|
||||
if check_docker_auth "$registry"; then
|
||||
echo "[docker-login] Already authenticated to $registry in this job"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Perform login to isolated config directory
|
||||
echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
|
||||
if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
|
||||
echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
|
||||
return 0
|
||||
else
|
||||
local exit_code=$?
|
||||
echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# Execute if run as script (not sourced)
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
safe_docker_login_ecr_public
|
||||
fi
|
||||
@@ -1,145 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script build the Ascend NPU docker image and run the offline inference inside the container.
|
||||
# It serves a sanity check for compilation and basic model usage.
|
||||
set -ex
|
||||
|
||||
# Base ubuntu image with basic ascend development libraries and python installed
|
||||
VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
|
||||
BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
|
||||
image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
|
||||
# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
|
||||
container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
|
||||
|
||||
# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
|
||||
agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
|
||||
echo "agent_idx: ${agent_idx}"
|
||||
builder_name="cachebuilder${agent_idx}"
|
||||
builder_cache_dir="/mnt/docker-cache${agent_idx}"
|
||||
mkdir -p ${builder_cache_dir}
|
||||
|
||||
# Try building the docker image
|
||||
cat <<EOF | DOCKER_BUILDKIT=1 docker build \
|
||||
--add-host pypi-cache:${PYPI_CACHE_HOST} \
|
||||
--builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
|
||||
--cache-to type=local,dest=${builder_cache_dir},mode=max \
|
||||
--build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
|
||||
--build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
|
||||
--progress=plain --load -t ${image_name} -f - .
|
||||
FROM ${BASE_IMAGE_NAME}
|
||||
|
||||
# Define environments
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
|
||||
pip config set global.trusted-host pypi-cache && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
|
||||
rm -rf /var/cache/apt/* && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install for pytest to make the docker build cache layer always valid
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install pytest>=6.0 pytest-cov modelscope
|
||||
|
||||
COPY . .
|
||||
|
||||
# Install vllm-omni
|
||||
WORKDIR /workspace
|
||||
ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
|
||||
ARG VLLM_OMNI_TAG=main
|
||||
ARG BUILDKITE_PULL_REQUEST
|
||||
ARG BUILDKITE_PULL_REQUEST_REPO
|
||||
RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
|
||||
if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
|
||||
echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
|
||||
git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
|
||||
cd /workspace/vllm-omni && \
|
||||
git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
|
||||
git checkout pr-\$BUILDKITE_PULL_REQUEST; \
|
||||
else \
|
||||
echo "Not a PR build, using main branch" && \
|
||||
git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
|
||||
fi
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh && \
|
||||
export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
|
||||
python3 -m pip install -v -e /workspace/vllm-omni/
|
||||
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
WORKDIR /workspace/vllm-omni
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
EOF
|
||||
|
||||
# Setup cleanup
|
||||
remove_docker_container() {
|
||||
docker rm -f "${container_name}" || true;
|
||||
docker image rm -f "${image_name}" || true;
|
||||
docker system prune -f || true;
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
|
||||
# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
|
||||
# e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
|
||||
# returns --device /dev/davinci0 --device /dev/davinci1
|
||||
parse_and_gen_devices() {
|
||||
local input="$1"
|
||||
local index cards_num
|
||||
if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
|
||||
index="${BASH_REMATCH[1]}"
|
||||
cards_num="${BASH_REMATCH[2]}"
|
||||
else
|
||||
echo "parse error" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local devices=""
|
||||
local i=0
|
||||
while (( i < cards_num )); do
|
||||
local dev_idx=$(((index - 1)*cards_num + i ))
|
||||
devices="$devices --device /dev/davinci${dev_idx}"
|
||||
((i++))
|
||||
done
|
||||
|
||||
# trim leading space
|
||||
devices="${devices#"${devices%%[![:space:]]*}"}"
|
||||
# Output devices: assigned to the caller variable
|
||||
printf '%s' "$devices"
|
||||
}
|
||||
|
||||
devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
|
||||
|
||||
# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
|
||||
# This test checks whether the OOT platform interface is functioning properly in conjunction with
|
||||
# the hardware plugin vllm-ascend.
|
||||
hf_model_cache_dir=/mnt/hf_cache${agent_idx}
|
||||
ms_model_cache_dir=/mnt/modelscope${agent_idx}
|
||||
mkdir -p ${hf_model_cache_dir}
|
||||
mkdir -p ${ms_model_cache_dir}
|
||||
docker run \
|
||||
--init \
|
||||
${devices} \
|
||||
--device /dev/davinci_manager \
|
||||
--device /dev/devmm_svm \
|
||||
--device /dev/hisi_hdc \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
|
||||
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v ${hf_model_cache_dir}:/root/.cache/huggingface \
|
||||
-v ${ms_model_cache_dir}:/root/.cache/modelscope \
|
||||
--network host \
|
||||
--entrypoint="" \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
bash -c '
|
||||
set -e
|
||||
VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
|
||||
'
|
||||
@@ -1,118 +0,0 @@
|
||||
steps:
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 30
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- pytest -s -v tests/e2e/offline_inference/test_z_image.py
|
||||
|
||||
- label: "Diffusion Images API LoRA E2E"
|
||||
timeout_in_minutes: 30
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
|
||||
|
||||
- label: "Diffusion Model CPU offloading Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
||||
- pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
||||
|
||||
- label: "Diffusion Cache Backend Test"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325'
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
|
||||
|
||||
- label: "Diffusion Tensor Parallelism Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
|
||||
|
||||
- label: "Diffusion GPU Worker Test"
|
||||
timeout_in_minutes: 20
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -s -v tests/diffusion/test_diffusion_worker.py
|
||||
|
||||
- label: "Omni Model Test Qwen2-5-Omni"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
|
||||
- pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
||||
|
||||
- label: "Omni Model Test Qwen3-Omni"
|
||||
timeout_in_minutes: 15
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
|
||||
|
||||
|
||||
- label: "Omni Sleep Mode Test"
|
||||
timeout_in_minutes: 40
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
|
||||
@@ -9,7 +9,7 @@ description: Generate and run tests for vllm-project/vllm-omni with CI-aligned l
|
||||
|
||||
Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
|
||||
|
||||
**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`).
|
||||
**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`).
|
||||
|
||||
Default priorities:
|
||||
|
||||
@@ -96,7 +96,7 @@ Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-styl
|
||||
|----------|------------------|---------------------------|---------------------------|
|
||||
| **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required |
|
||||
| **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` |
|
||||
| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
|
||||
| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/cuda/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
|
||||
| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps |
|
||||
| **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) |
|
||||
|
||||
@@ -460,7 +460,7 @@ When the user asks for **invalid parameter validation**, **invalid request bodie
|
||||
|
||||
4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_<model>.py` unless the route is new.
|
||||
|
||||
5. **CI — `.buildkite/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
|
||||
5. **CI — `.buildkite/cuda/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
|
||||
|
||||
| Weekly step | Command | When your cases run |
|
||||
|-------------|---------|---------------------|
|
||||
@@ -629,10 +629,10 @@ If the test is not already collected by an existing pipeline command (for exampl
|
||||
|
||||
| Test level | Edit this file | Typical trigger / intent |
|
||||
|------------|----------------|---------------------------|
|
||||
| **L1** and **L2** | [`.buildkite/test-ready.yml`](../../../.buildkite/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
|
||||
| **L3** | [`.buildkite/test-merge.yml`](../../../.buildkite/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
|
||||
| **L4** | [`.buildkite/test-nightly.yml`](../../../.buildkite/test-nightly.yml) | Nightly; grouped by model type (see below) |
|
||||
| **Invalid param / reliability (weekly)** | [`.buildkite/test-weekly.yml`](../../../.buildkite/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
|
||||
| **L1** and **L2** | [`.buildkite/cuda/test-ready.yml`](../../../.buildkite/cuda/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
|
||||
| **L3** | [`.buildkite/cuda/test-merge.yml`](../../../.buildkite/cuda/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
|
||||
| **L4** | [`.buildkite/cuda/test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) | Nightly; grouped by model type (see below) |
|
||||
| **Invalid param / reliability (weekly)** | [`.buildkite/cuda/test-weekly.yml`](../../../.buildkite/cuda/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
|
||||
|
||||
**Level-specific Buildkite delivery (follow the user’s requested level):**
|
||||
|
||||
@@ -809,21 +809,21 @@ Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`..
|
||||
|
||||
When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan.
|
||||
|
||||
#### L3 / L4: validating Buildkite from a feature branch (root `pipeline.yml`)
|
||||
#### L3 / L4: validating Buildkite from a feature branch (`cuda/pipeline.yml`)
|
||||
|
||||
Production behavior is driven by [`.buildkite/pipeline.yml`](../../../.buildkite/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
|
||||
Production behavior is driven by [`.buildkite/cuda/pipeline.yml`](../../../.buildkite/cuda/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
|
||||
|
||||
When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge):
|
||||
|
||||
1. **Point the upload at the right file (optional shortcut)**
|
||||
In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/test-ready.yml`. For a one-off run you may change that path to `.buildkite/test-merge.yml` (L3) or `.buildkite/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
|
||||
In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/cuda/test-ready.yml`. For a one-off run you may change that path to `.buildkite/cuda/test-merge.yml` (L3) or `.buildkite/cuda/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
|
||||
|
||||
2. **L3 — merge pipeline**
|
||||
The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
|
||||
The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `cuda/pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
|
||||
|
||||
3. **L4 — nightly pipeline**
|
||||
**Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `pipeline.yml` so the nightly definition is uploaded without setting the env var.
|
||||
Child steps in [`test-nightly.yml`](../../../.buildkite/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
|
||||
**Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `cuda/pipeline.yml` so the nightly definition is uploaded without setting the env var.
|
||||
Child steps in [`test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
|
||||
|
||||
### Step 6: Run Tests
|
||||
|
||||
@@ -890,5 +890,5 @@ When completing a request, return:
|
||||
## Additional Resources
|
||||
|
||||
- Marker and command routing: [references/test-routing.md](references/test-routing.md)
|
||||
- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/test-weekly.yml) (invalid param / reliability)
|
||||
- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/cuda/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/cuda/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/cuda/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/cuda/test-weekly.yml) (invalid param / reliability)
|
||||
- L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910)
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Use this reference to map testing goals to levels, markers, and runnable commands.
|
||||
|
||||
**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`).
|
||||
**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`).
|
||||
|
||||
## Model-centric e2e filename convention (L2–L4)
|
||||
|
||||
@@ -224,7 +224,7 @@ When the user’s test plan includes **invalid parameter validation**, **invalid
|
||||
| Sleep / wakeup / server control | `test_invalid_server_control.py` |
|
||||
|
||||
3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.<type>]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649).
|
||||
4. **CI:** [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
|
||||
4. **CI:** [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
|
||||
|
||||
```bash
|
||||
# Weekly H100 (diffusion / omni / video invalid-param)
|
||||
@@ -291,10 +291,10 @@ When adding or modifying tests, do not stop at “where the file lives” — al
|
||||
|
||||
| Level | Repo file | Model-type grouping |
|
||||
|-------|-----------|---------------------|
|
||||
| L1, L2 | [`.buildkite/test-ready.yml`](../../../../.buildkite/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
|
||||
| L3 | [`.buildkite/test-merge.yml`](../../../../.buildkite/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
|
||||
| L4 | [`.buildkite/test-nightly.yml`](../../../../.buildkite/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
|
||||
| Invalid param (weekly) | [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
|
||||
| L1, L2 | [`.buildkite/cuda/test-ready.yml`](../../../../.buildkite/cuda/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
|
||||
| L3 | [`.buildkite/cuda/test-merge.yml`](../../../../.buildkite/cuda/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
|
||||
| L4 | [`.buildkite/cuda/test-nightly.yml`](../../../../.buildkite/cuda/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
|
||||
| Invalid param (weekly) | [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
|
||||
|
||||
### `source_file_dependencies` (E2E Test only — ready & merge)
|
||||
|
||||
@@ -380,7 +380,7 @@ Common patterns in `test-ready.yml` / `test-merge.yml`:
|
||||
|
||||
When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes.
|
||||
|
||||
The root [`.buildkite/pipeline.yml`](../../../../.buildkite/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
|
||||
[`.buildkite/cuda/pipeline.yml`](../../../../.buildkite/cuda/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
|
||||
|
||||
Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`.
|
||||
|
||||
|
||||
@@ -131,7 +131,7 @@ Please run the L1 and L2 test cases locally first and attach the results before
|
||||
|
||||
### Automatic skip-ci (docs and pytest skip marks)
|
||||
|
||||
On pull requests and `main` pushes, the bootstrap step in [`.buildkite/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/pipeline.yml) runs [`.buildkite/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
|
||||
On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/cuda/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
|
||||
|
||||
| Change per file | Examples |
|
||||
| --- | --- |
|
||||
|
||||
@@ -319,7 +319,7 @@ Before entering specific testing levels, the project establishes two common spec
|
||||
|
||||
### Diff-aware Buildkite uploads (`source_file_dependencies`)
|
||||
|
||||
L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`.
|
||||
L2 (`.buildkite/cuda/test-ready.yml`) and L3 (`.buildkite/cuda/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/cuda/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`.
|
||||
|
||||
#### What `source_file_dependencies` is
|
||||
|
||||
@@ -334,14 +334,14 @@ L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines
|
||||
| `main` branch push | `git diff --name-only <commit>^..<commit>` |
|
||||
| Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped |
|
||||
|
||||
Docs-only PRs are handled earlier in bootstrap (`pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines.
|
||||
Docs-only PRs are handled earlier in bootstrap (`.buildkite/cuda/pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines.
|
||||
|
||||
#### Where it is configured
|
||||
|
||||
| Level | Pipeline file | Upload entry |
|
||||
| --- | --- | --- |
|
||||
| L2 | `.buildkite/test-ready.yml` | `upload_pipeline.py --upload .buildkite/test-ready.yml` (from `pipeline.yml`) |
|
||||
| L3 | `.buildkite/test-merge.yml` | `upload_pipeline.py --upload .buildkite/test-merge.yml` (from `pipeline.yml`) |
|
||||
| L2 | `.buildkite/cuda/test-ready.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-ready.yml` (from `cuda/pipeline.yml`) |
|
||||
| L3 | `.buildkite/cuda/test-merge.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-merge.yml` (from `cuda/pipeline.yml`) |
|
||||
|
||||
Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3).
|
||||
|
||||
@@ -393,10 +393,10 @@ A **group** may also define `source_file_dependencies`; nested steps inherit fil
|
||||
|
||||
```bash
|
||||
# Render filtered YAML to stdout (no upload)
|
||||
python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-ready.yml
|
||||
python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml
|
||||
|
||||
# Confirm uploader-only keys are stripped
|
||||
python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-merge.yml | grep source_file_dependencies
|
||||
python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies
|
||||
# (no output expected)
|
||||
```
|
||||
|
||||
@@ -404,7 +404,7 @@ On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `s
|
||||
|
||||
#### Related
|
||||
|
||||
- Implementation: `.buildkite/scripts/upload_pipeline.py`
|
||||
- Implementation: `.buildkite/cuda/scripts/upload_pipeline.py`
|
||||
- L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled.
|
||||
|
||||
### Test helper environment variables
|
||||
@@ -871,7 +871,7 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in
|
||||
|
||||
#### CI trigger
|
||||
|
||||
Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
|
||||
Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
|
||||
|
||||
| Buildkite step | Test file | CI hardware |
|
||||
| -------------- | --------- | ----------- |
|
||||
@@ -902,12 +902,12 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
1. Add `test_reliability_<model>.py` under `tests/dfx/reliability/`.
|
||||
2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files).
|
||||
3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML.
|
||||
4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/test-weekly.yml`** when the suite should run in weekly L5.
|
||||
4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/cuda/test-weekly.yml`** when the suite should run in weekly L5.
|
||||
|
||||
</details>
|
||||
|
||||
- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_<suite>.py` and a matching step in `.buildkite/test-weekly.yml` for new models)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_<suite>.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models)
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -49,11 +49,11 @@ Failed jobs: 1/2
|
||||
cd tests
|
||||
pytest -s -v -m "core_model and cpu"
|
||||
```
|
||||
The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml).
|
||||
The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml).
|
||||
|
||||
=== "L2 level"
|
||||
|
||||
**Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
**Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/cuda/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
|
||||
```bash
|
||||
# All L2 and L1 jobs (default: every pytest step in test-ready.yml)
|
||||
@@ -85,7 +85,7 @@ Failed jobs: 1/2
|
||||
|
||||
=== "L3 level"
|
||||
|
||||
**Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
**Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/cuda/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
|
||||
```bash
|
||||
# All L3 and L1 jobs (default: every pytest step in test-merge.yml)
|
||||
@@ -117,7 +117,7 @@ Failed jobs: 1/2
|
||||
|
||||
=== "L4 level"
|
||||
|
||||
**Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
**Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/cuda/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`).
|
||||
|
||||
```bash
|
||||
# All nightly jobs (default: test-type all, model-type all)
|
||||
@@ -189,7 +189,7 @@ Failed jobs: 1/2
|
||||
|
||||
```
|
||||
|
||||
The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml).
|
||||
The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml).
|
||||
|
||||
You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b
|
||||
|
||||
## Source of Conclusions
|
||||
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`.
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`.
|
||||
|
||||
### Example: `SIGTERM` fault injection log (Qwen3-Omni)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Extract steps from .buildkite/test-nightly.yml that contain pytest, synthesize
|
||||
# Extract steps from .buildkite/cuda/test-nightly.yml that contain pytest, synthesize
|
||||
# small bash wrappers (exports + pytest), run them, and tee output to logs named
|
||||
# after each step's Buildkite "key" when present (otherwise a slug of the label).
|
||||
# YAML steps whose label contains "Perf Test" run first, then
|
||||
@@ -51,17 +51,17 @@
|
||||
# Usage:
|
||||
# bash path/to/run_nightly_jobs.sh
|
||||
# REPO_ROOT=/path/to/vllm-omni bash path/to/run_nightly_jobs.sh --test-type perf,acc --dry-run
|
||||
# YML=/path/to/vllm-omni/.buildkite/test-nightly.yml bash path/to/run_nightly_jobs.sh
|
||||
# YML=/path/to/vllm-omni/.buildkite/cuda/test-nightly.yml bash path/to/run_nightly_jobs.sh
|
||||
#
|
||||
# Repository / YAML (no dependency on where this script lives):
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-nightly.yml
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-nightly.yml
|
||||
# • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory
|
||||
# • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD,
|
||||
# then from the script's directory, until .buildkite/test-nightly.yml exists
|
||||
# then from the script's directory, until .buildkite/cuda/test-nightly.yml exists
|
||||
#
|
||||
# Optional environment:
|
||||
# REPO_ROOT - vllm-omni root (working directory for pytest); see above
|
||||
# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/test-nightly.yml)
|
||||
# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/cuda/test-nightly.yml)
|
||||
# LOG_DIR - logs + generated job scripts; when unset, a timestamped directory under
|
||||
# $REPO_ROOT/logs/ is created:
|
||||
# nightly_jobs_YYYYMMDD-HHMMSS (default / YAML nightly steps)
|
||||
@@ -247,7 +247,7 @@ _nightly_log_dir_basename() {
|
||||
fi
|
||||
}
|
||||
|
||||
BUILDKITE_REL=".buildkite/test-nightly.yml"
|
||||
BUILDKITE_REL=".buildkite/cuda/test-nightly.yml"
|
||||
|
||||
_find_repo_containing_nightly() {
|
||||
local dir="${1:-}"
|
||||
@@ -267,8 +267,14 @@ _derive_repo_root_from_yml() {
|
||||
local yml="$1"
|
||||
local d
|
||||
d="$(cd "$(dirname "${yml}")" && pwd)"
|
||||
[[ "$(basename "${d}")" == ".buildkite" ]] || return 1
|
||||
printf '%s\n' "$(dirname "${d}")"
|
||||
while [[ "${d}" != "/" ]]; do
|
||||
if [[ "$(basename "${d}")" == ".buildkite" ]]; then
|
||||
printf '%s\n' "$(dirname "${d}")"
|
||||
return 0
|
||||
fi
|
||||
d="$(dirname "${d}")"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Resolve REPO_ROOT, YML (no relative path between script and repo)
|
||||
@@ -278,7 +284,7 @@ if [[ -n "${YML:-}" && -n "${REPO_ROOT:-}" ]]; then
|
||||
elif [[ -n "${YML:-}" ]]; then
|
||||
YML="$(cd "$(dirname "${YML}")" && pwd)/$(basename "${YML}")"
|
||||
if ! REPO_ROOT="$(_derive_repo_root_from_yml "${YML}")"; then
|
||||
echo "Could not derive REPO_ROOT from YML=${YML} (expected file at <repo>/.buildkite/test-nightly.yml)." >&2
|
||||
echo "Could not derive REPO_ROOT from YML=${YML} (expected file at <repo>/.buildkite/cuda/test-nightly.yml)." >&2
|
||||
echo "Set REPO_ROOT explicitly (or pass --repo-root) for pytest working directory." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -181,7 +181,7 @@ _run_jobs_print_timing_summary() {
|
||||
# tools/run_merge_jobs.sh). Nightly does not use this entrypoint.
|
||||
#
|
||||
# Entry scripts must set before calling run_yaml_ci_jobs_main:
|
||||
# BUILDKITE_REL - e.g. .buildkite/test-ready.yml
|
||||
# BUILDKITE_REL - e.g. .buildkite/cuda/test-ready.yml
|
||||
# DEFAULT_LOG_SUBDIR - e.g. ready_jobs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -246,8 +246,14 @@ _run_jobs_derive_repo_root_from_yml() {
|
||||
local yml="$1"
|
||||
local d
|
||||
d="$(cd "$(dirname "${yml}")" && pwd)"
|
||||
[[ "$(basename "${d}")" == ".buildkite" ]] || return 1
|
||||
printf '%s\n' "$(dirname "${d}")"
|
||||
while [[ "${d}" != "/" ]]; do
|
||||
if [[ "$(basename "${d}")" == ".buildkite" ]]; then
|
||||
printf '%s\n' "$(dirname "${d}")"
|
||||
return 0
|
||||
fi
|
||||
d="$(dirname "${d}")"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_yaml_ci_jobs_main() {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Extract steps from .buildkite/test-merge.yml that contain pytest, synthesize
|
||||
# Extract steps from .buildkite/cuda/test-merge.yml that contain pytest, synthesize
|
||||
# small bash wrappers (exports + pytest), run them, and tee output to logs named
|
||||
# after each step's Buildkite "key" when present (otherwise a slug of the label).
|
||||
#
|
||||
@@ -21,17 +21,17 @@
|
||||
# Usage:
|
||||
# bash tools/run_merge_jobs.sh
|
||||
# REPO_ROOT=/path/to/vllm-omni bash tools/run_merge_jobs.sh --model-type diffusion --dry-run
|
||||
# YML=/path/to/vllm-omni/.buildkite/test-merge.yml bash tools/run_merge_jobs.sh
|
||||
# YML=/path/to/vllm-omni/.buildkite/cuda/test-merge.yml bash tools/run_merge_jobs.sh
|
||||
#
|
||||
# Repository / YAML (no dependency on where this script lives):
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-merge.yml
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-merge.yml
|
||||
# • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory
|
||||
# • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD,
|
||||
# then from the script's directory, until .buildkite/test-merge.yml exists
|
||||
# then from the script's directory, until .buildkite/cuda/test-merge.yml exists
|
||||
#
|
||||
# Optional environment:
|
||||
# REPO_ROOT - vllm-omni root (working directory for pytest); see above
|
||||
# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/test-merge.yml)
|
||||
# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/cuda/test-merge.yml)
|
||||
# LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/merge_jobs);
|
||||
# per-job *.log plus timing_summary.log after the run
|
||||
# MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above
|
||||
@@ -41,7 +41,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BUILDKITE_REL=".buildkite/test-merge.yml"
|
||||
BUILDKITE_REL=".buildkite/cuda/test-merge.yml"
|
||||
DEFAULT_LOG_SUBDIR="merge_jobs"
|
||||
|
||||
# shellcheck source=tools/run_jobs_common.sh
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Extract steps from .buildkite/test-ready.yml that contain pytest, synthesize
|
||||
# Extract steps from .buildkite/cuda/test-ready.yml that contain pytest, synthesize
|
||||
# small bash wrappers (exports + pytest), run them, and tee output to logs named
|
||||
# after each step's Buildkite "key" when present (otherwise a slug of the label).
|
||||
#
|
||||
@@ -21,17 +21,17 @@
|
||||
# Usage:
|
||||
# bash tools/run_ready_jobs.sh
|
||||
# REPO_ROOT=/path/to/vllm-omni bash tools/run_ready_jobs.sh --model-type omni --dry-run
|
||||
# YML=/path/to/vllm-omni/.buildkite/test-ready.yml bash tools/run_ready_jobs.sh
|
||||
# YML=/path/to/vllm-omni/.buildkite/cuda/test-ready.yml bash tools/run_ready_jobs.sh
|
||||
#
|
||||
# Repository / YAML (no dependency on where this script lives):
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-ready.yml
|
||||
# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-ready.yml
|
||||
# • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory
|
||||
# • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD,
|
||||
# then from the script's directory, until .buildkite/test-ready.yml exists
|
||||
# then from the script's directory, until .buildkite/cuda/test-ready.yml exists
|
||||
#
|
||||
# Optional environment:
|
||||
# REPO_ROOT - vllm-omni root (working directory for pytest); see above
|
||||
# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/test-ready.yml)
|
||||
# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/cuda/test-ready.yml)
|
||||
# LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/ready_jobs);
|
||||
# per-job *.log plus timing_summary.log after the run
|
||||
# MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above
|
||||
@@ -41,7 +41,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BUILDKITE_REL=".buildkite/test-ready.yml"
|
||||
BUILDKITE_REL=".buildkite/cuda/test-ready.yml"
|
||||
DEFAULT_LOG_SUBDIR="ready_jobs"
|
||||
|
||||
# shellcheck source=tools/run_jobs_common.sh
|
||||
|
||||
Reference in new issue
Block a user