From 0d8c63268896bebeb473e396ebbc882f0994a041 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:06:54 +0800 Subject: [PATCH] [Refactor][CI] Reorganize .buildkite directory by platform (#5119) --- .../{ => amd/scripts}/bootstrap-amd-omni.sh | 14 +- .../scripts}/run-amd-test.sh | 2 +- .buildkite/{ => amd}/test-amd-merge.yml | 0 .../test-amd-ready.yml} | 0 .../{ => amd}/test-template-amd-omni.j2 | 4 +- .buildkite/{ => cuda}/pipeline.yml | 10 +- .../rebase-pipeline.yml} | 6 +- .../{ => cuda}/scripts/upload_pipeline.py | 16 +- .buildkite/{ => cuda}/test-merge.yml | 0 .buildkite/{ => cuda}/test-nightly.yml | 0 .buildkite/{ => cuda}/test-ready.yml | 0 .buildkite/{ => cuda}/test-weekly.yml | 0 .../pipeline-intel.yml} | 4 +- .../scripts}/bootstrap-intel-omni.sh | 2 +- .../scripts}/run-xpu-test.sh | 0 .../pipeline-npu-a3.yml} | 4 +- .../pipeline-npu.yml} | 4 +- .../test-npu-nightly.yml} | 0 .../test-npu-ready.yml} | 0 .../release-pipeline.yml} | 8 +- .../generate-and-upload-nightly-index.sh | 6 +- .../scripts/generate-nightly-index.py | 0 .../scripts/publish-release-images.sh | 0 .../scripts/publish-release-wheels-pypi.sh | 0 .../scripts/upload-nightly-wheels.sh | 0 .buildkite/scripts/docker_login_ecr_public.sh | 73 --------- .../scripts/hardware_ci/run_npu_test.sh | 145 ------------------ .buildkite/test-amd.yaml | 118 -------------- .claude/skills/vllm-omni-test/SKILL.md | 28 ++-- .../vllm-omni-test/references/test-routing.md | 14 +- docs/contributing/README.md | 2 +- docs/contributing/ci/CI_5levels.md | 20 +-- docs/contributing/ci/test_guide.md | 10 +- .../fault_injection_reliability_matrix.md | 2 +- tools/nightly/run_nightly_jobs.sh | 24 +-- tools/run_jobs_common.sh | 12 +- tools/run_merge_jobs.sh | 12 +- tools/run_ready_jobs.sh | 12 +- 38 files changed, 114 insertions(+), 438 deletions(-) rename .buildkite/{ => amd/scripts}/bootstrap-amd-omni.sh (94%) rename .buildkite/{scripts/hardware_ci => amd/scripts}/run-amd-test.sh (98%) rename .buildkite/{ => amd}/test-amd-merge.yml (100%) rename .buildkite/{test-amd-ready.yaml => amd/test-amd-ready.yml} (100%) rename .buildkite/{ => amd}/test-template-amd-omni.j2 (96%) rename .buildkite/{ => cuda}/pipeline.yml (87%) rename .buildkite/{rebase-pipeline.yaml => cuda/rebase-pipeline.yml} (76%) rename .buildkite/{ => cuda}/scripts/upload_pipeline.py (97%) rename .buildkite/{ => cuda}/test-merge.yml (100%) rename .buildkite/{ => cuda}/test-nightly.yml (100%) rename .buildkite/{ => cuda}/test-ready.yml (100%) rename .buildkite/{ => cuda}/test-weekly.yml (100%) rename .buildkite/{pipeline-intel.yaml => intel/pipeline-intel.yml} (79%) rename .buildkite/{ => intel/scripts}/bootstrap-intel-omni.sh (94%) rename .buildkite/{scripts/hardware_ci => intel/scripts}/run-xpu-test.sh (100%) rename .buildkite/{pipeline-npu-a3.yaml => npu/pipeline-npu-a3.yml} (97%) rename .buildkite/{pipeline-npu.yaml => npu/pipeline-npu.yml} (98%) rename .buildkite/{test-npu-nightly.yaml => npu/test-npu-nightly.yml} (100%) rename .buildkite/{test-npu-ready.yaml => npu/test-npu-ready.yml} (100%) rename .buildkite/{release-pipeline.yaml => release/release-pipeline.yml} (92%) rename .buildkite/{ => release}/scripts/generate-and-upload-nightly-index.sh (92%) rename .buildkite/{ => release}/scripts/generate-nightly-index.py (100%) rename .buildkite/{ => release}/scripts/publish-release-images.sh (100%) rename .buildkite/{ => release}/scripts/publish-release-wheels-pypi.sh (100%) rename .buildkite/{ => release}/scripts/upload-nightly-wheels.sh (100%) delete mode 100755 .buildkite/scripts/docker_login_ecr_public.sh delete mode 100644 .buildkite/scripts/hardware_ci/run_npu_test.sh delete mode 100644 .buildkite/test-amd.yaml diff --git a/.buildkite/bootstrap-amd-omni.sh b/.buildkite/amd/scripts/bootstrap-amd-omni.sh similarity index 94% rename from .buildkite/bootstrap-amd-omni.sh rename to .buildkite/amd/scripts/bootstrap-amd-omni.sh index 16ed02b26..ee858068a 100755 --- a/.buildkite/bootstrap-amd-omni.sh +++ b/.buildkite/amd/scripts/bootstrap-amd-omni.sh @@ -154,14 +154,14 @@ upload_pipeline() { fi # Use local template file for vllm-omni - cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2 + cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2 # (WIP) Use pipeline generator instead of jinja template - if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then + if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then python -m pip install click pydantic - python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW" - buildkite-agent pipeline upload .buildkite/pipeline.yaml + python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW" + buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml exit 0 fi echo "List file diff: $LIST_FILE_DIFF" @@ -171,7 +171,7 @@ upload_pipeline() { FAIL_FAST=$(fail_fast) - cd .buildkite + cd .buildkite/amd # Select test definition file: merge suite for main, ready suite for PRs. # For debugging, DEBUG_TEST_YAML can override the selection — accepts @@ -182,7 +182,7 @@ upload_pipeline() { TEST_YAML="test-amd-merge.yml" ;; ready) - TEST_YAML="test-amd-ready.yaml" + TEST_YAML="test-amd-ready.yml" ;; *) echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2 @@ -193,7 +193,7 @@ upload_pipeline() { elif [[ $BUILDKITE_BRANCH == "main" ]]; then TEST_YAML="test-amd-merge.yml" else - TEST_YAML="test-amd-ready.yaml" + TEST_YAML="test-amd-ready.yml" fi ( diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/amd/scripts/run-amd-test.sh similarity index 98% rename from .buildkite/scripts/hardware_ci/run-amd-test.sh rename to .buildkite/amd/scripts/run-amd-test.sh index f880455d4..c49614e70 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/amd/scripts/run-amd-test.sh @@ -56,7 +56,7 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea # # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images # sudo apt-get update && sudo apt-get install -y awscli ## Use safe docker login helper to prevent race conditions -# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh" +# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh" # safe_docker_login_ecr_public ## Pull the container from AMD Docker Hub diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml similarity index 100% rename from .buildkite/test-amd-merge.yml rename to .buildkite/amd/test-amd-merge.yml diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/amd/test-amd-ready.yml similarity index 100% rename from .buildkite/test-amd-ready.yaml rename to .buildkite/amd/test-amd-ready.yml diff --git a/.buildkite/test-template-amd-omni.j2 b/.buildkite/amd/test-template-amd-omni.j2 similarity index 96% rename from .buildkite/test-template-amd-omni.j2 rename to .buildkite/amd/test-template-amd-omni.j2 index aa5ca8bcd..2f1a9aed3 100644 --- a/.buildkite/test-template-amd-omni.j2 +++ b/.buildkite/amd/test-template-amd-omni.j2 @@ -57,7 +57,7 @@ steps: {% endif %} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh + command: bash .buildkite/amd/scripts/run-amd-test.sh env: DOCKER_BUILDKIT: "1" {% if step.env %} @@ -104,7 +104,7 @@ steps: {% endif %} {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} {% set indented_cmd = cmd_body | replace("\n", "\n ") %} - command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh + command: bash .buildkite/amd/scripts/run-amd-test.sh env: DOCKER_BUILDKIT: "1" {% if step.env %} diff --git a/.buildkite/pipeline.yml b/.buildkite/cuda/pipeline.yml similarity index 87% rename from .buildkite/pipeline.yml rename to .buildkite/cuda/pipeline.yml index 040393f6c..3b460c1c6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/cuda/pipeline.yml @@ -9,7 +9,7 @@ steps: - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline" key: upload-ci-pipeline commands: - - "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml" + - "python3 .buildkite/cuda/scripts/upload_pipeline.py --upload .buildkite/cuda/pipeline.yml" agents: queue: "cpu_queue_premerge" @@ -37,7 +37,7 @@ steps: if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi - python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-ready.yml + python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -52,7 +52,7 @@ steps: if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi - python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-merge.yml + python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -70,7 +70,7 @@ steps: build.pull_request.labels includes "diffusion-x2v-test" )) commands: - - buildkite-agent pipeline upload .buildkite/test-nightly.yml + - buildkite-agent pipeline upload .buildkite/cuda/test-nightly.yml agents: queue: "cpu_queue_premerge" @@ -82,6 +82,6 @@ steps: (build.branch == "main" && build.env("WEEKLY") == "1") || (build.branch != "main" && build.pull_request.labels includes "weekly-test") commands: - - buildkite-agent pipeline upload .buildkite/test-weekly.yml + - buildkite-agent pipeline upload .buildkite/cuda/test-weekly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/cuda/rebase-pipeline.yml similarity index 76% rename from .buildkite/rebase-pipeline.yaml rename to .buildkite/cuda/rebase-pipeline.yml index 796ad19a7..514c9cc1e 100644 --- a/.buildkite/rebase-pipeline.yaml +++ b/.buildkite/cuda/rebase-pipeline.yml @@ -14,7 +14,7 @@ steps: depends_on: image-build key: upload-ready-pipeline commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -23,7 +23,7 @@ steps: depends_on: image-build key: upload-merge-pipeline commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -33,6 +33,6 @@ steps: key: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" commands: - - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml + - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/scripts/upload_pipeline.py b/.buildkite/cuda/scripts/upload_pipeline.py similarity index 97% rename from .buildkite/scripts/upload_pipeline.py rename to .buildkite/cuda/scripts/upload_pipeline.py index d9220ed0f..dc233af7a 100644 --- a/.buildkite/scripts/upload_pipeline.py +++ b/.buildkite/cuda/scripts/upload_pipeline.py @@ -16,16 +16,16 @@ Usage: python3 upload_pipeline.py [--upload] [--all | --e2e] # Bootstrap (replaces upload_pipeline_with_skip_ci.sh): - python3 upload_pipeline.py --upload .buildkite/pipeline.yml + python3 upload_pipeline.py --upload .buildkite/cuda/pipeline.yml # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py): - python3 upload_pipeline.py --upload .buildkite/test-merge.yml + python3 upload_pipeline.py --upload .buildkite/cuda/test-merge.yml # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key: - python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml + python3 upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml # Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies: - python3 upload_pipeline.py --upload --e2e .buildkite/test-ready.yml + python3 upload_pipeline.py --upload --e2e .buildkite/cuda/test-merge.yml """ from __future__ import annotations @@ -47,7 +47,7 @@ except ImportError: import yaml LOG = "upload_pipeline" -ROOT = Path(__file__).resolve().parent.parent.parent +ROOT = Path(__file__).resolve().parent.parent.parent.parent DOC_SEP = "\n---\n" BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__" E2E_GROUP_MARKER = "E2E Test" @@ -475,7 +475,7 @@ def render_pipeline( # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds - # run the full e2e suite (see .buildkite/rebase-pipeline.yaml). + # run the full e2e suite (see .buildkite/cuda/rebase-pipeline.yml). if force_all or e2e_only: changed_files = None @@ -509,8 +509,8 @@ def main() -> int: parser.add_argument( "pipeline", nargs="?", - default=".buildkite/pipeline.yml", - help="Pipeline YAML path (default: .buildkite/pipeline.yml)", + default=".buildkite/cuda/pipeline.yml", + help="Pipeline YAML path (default: .buildkite/cuda/pipeline.yml)", ) parser.add_argument( "--upload", diff --git a/.buildkite/test-merge.yml b/.buildkite/cuda/test-merge.yml similarity index 100% rename from .buildkite/test-merge.yml rename to .buildkite/cuda/test-merge.yml diff --git a/.buildkite/test-nightly.yml b/.buildkite/cuda/test-nightly.yml similarity index 100% rename from .buildkite/test-nightly.yml rename to .buildkite/cuda/test-nightly.yml diff --git a/.buildkite/test-ready.yml b/.buildkite/cuda/test-ready.yml similarity index 100% rename from .buildkite/test-ready.yml rename to .buildkite/cuda/test-ready.yml diff --git a/.buildkite/test-weekly.yml b/.buildkite/cuda/test-weekly.yml similarity index 100% rename from .buildkite/test-weekly.yml rename to .buildkite/cuda/test-weekly.yml diff --git a/.buildkite/pipeline-intel.yaml b/.buildkite/intel/pipeline-intel.yml similarity index 79% rename from .buildkite/pipeline-intel.yaml rename to .buildkite/intel/pipeline-intel.yml index d3a2655bf..e68f59e53 100644 --- a/.buildkite/pipeline-intel.yaml +++ b/.buildkite/intel/pipeline-intel.yml @@ -1,9 +1,9 @@ -# .buildkite/pipeline-intel.yaml +# .buildkite/intel/pipeline-intel.yml steps: - group: "INTEL Tests" steps: - label: "Intel: XPU Test" - command: "bash .buildkite/scripts/hardware_ci/run-xpu-test.sh" + command: "bash .buildkite/intel/scripts/run-xpu-test.sh" agents: queue: "intel-gpu-omni" env: diff --git a/.buildkite/bootstrap-intel-omni.sh b/.buildkite/intel/scripts/bootstrap-intel-omni.sh similarity index 94% rename from .buildkite/bootstrap-intel-omni.sh rename to .buildkite/intel/scripts/bootstrap-intel-omni.sh index b99568141..e51252bd9 100755 --- a/.buildkite/bootstrap-intel-omni.sh +++ b/.buildkite/intel/scripts/bootstrap-intel-omni.sh @@ -11,7 +11,7 @@ fi upload_pipeline() { echo "--- 🛠 Preparing Intel pipeline" - buildkite-agent pipeline upload .buildkite/pipeline-intel.yaml + buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml } get_diff() { diff --git a/.buildkite/scripts/hardware_ci/run-xpu-test.sh b/.buildkite/intel/scripts/run-xpu-test.sh similarity index 100% rename from .buildkite/scripts/hardware_ci/run-xpu-test.sh rename to .buildkite/intel/scripts/run-xpu-test.sh diff --git a/.buildkite/pipeline-npu-a3.yaml b/.buildkite/npu/pipeline-npu-a3.yml similarity index 97% rename from .buildkite/pipeline-npu-a3.yaml rename to .buildkite/npu/pipeline-npu-a3.yml index 8776219e1..a57f5f8e5 100644 --- a/.buildkite/pipeline-npu-a3.yaml +++ b/.buildkite/npu/pipeline-npu-a3.yml @@ -88,7 +88,7 @@ # # queue: "ascend-a3" # # resource_class: "npu-2" # # command: | -# # buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml +# # buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) # - label: "Upload Nightly Pipeline" @@ -104,7 +104,7 @@ # build.pull_request.labels includes "diffusion-x2v-test" # )) # commands: - # - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml + # - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml # agents: # queue: "ascend-a3" # resource_class: "npu-2" diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/npu/pipeline-npu.yml similarity index 98% rename from .buildkite/pipeline-npu.yaml rename to .buildkite/npu/pipeline-npu.yml index d035ffc94..fd67711f0 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/npu/pipeline-npu.yml @@ -171,7 +171,7 @@ steps: queue: "ascend-a2b3" resource_class: "npu-1" command: | - buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml + buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - label: "Upload Nightly Pipeline" @@ -189,7 +189,7 @@ steps: build.pull_request.labels includes "diffusion-x2v-test" )) commands: - - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml + - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml agents: queue: "ascend-a2b3" resource_class: "npu-1" diff --git a/.buildkite/test-npu-nightly.yaml b/.buildkite/npu/test-npu-nightly.yml similarity index 100% rename from .buildkite/test-npu-nightly.yaml rename to .buildkite/npu/test-npu-nightly.yml diff --git a/.buildkite/test-npu-ready.yaml b/.buildkite/npu/test-npu-ready.yml similarity index 100% rename from .buildkite/test-npu-ready.yaml rename to .buildkite/npu/test-npu-ready.yml diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release/release-pipeline.yml similarity index 92% rename from .buildkite/release-pipeline.yaml rename to .buildkite/release/release-pipeline.yml index c560fefe1..351e105cc 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release/release-pipeline.yml @@ -16,7 +16,7 @@ steps: - "uv venv --python=3.12 && source .venv/bin/activate" - "uv pip install --upgrade build" - "python3 -m build" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - "bash .buildkite/release/scripts/upload-nightly-wheels.sh" - label: "Generate and upload wheel indices" depends_on: "build-wheel" @@ -24,7 +24,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" + - "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh" - block: "Upload release wheels to PyPI" depends_on: "build-wheel" @@ -38,7 +38,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/publish-release-wheels-pypi.sh" + - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh" - block: "Unblock to build release Docker images" depends_on: ~ @@ -100,7 +100,7 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/publish-release-images.sh" + - "bash .buildkite/release/scripts/publish-release-images.sh" plugins: - docker-login#v3.0.0: username: vllmbot diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/release/scripts/generate-and-upload-nightly-index.sh similarity index 92% rename from .buildkite/scripts/generate-and-upload-nightly-index.sh rename to .buildkite/release/scripts/generate-and-upload-nightly-index.sh index b09c13f5c..63f5da596 100755 --- a/.buildkite/scripts/generate-and-upload-nightly-index.sh +++ b/.buildkite/release/scripts/generate-and-upload-nightly-index.sh @@ -36,12 +36,12 @@ mkdir -p "$INDICES_OUTPUT_DIR" # HACK: we do not need regex module here, but it is required by pre-commit hook # To avoid any external dependency, we simply replace it back to the stdlib re module -sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak +sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak # Generate indices -- the version is just the commit hash (not omni/{commit}) # because relative paths are computed between the index and wheel directories, # both of which live under the omni/ prefix in S3. -$PYTHON .buildkite/scripts/generate-nightly-index.py \ +$PYTHON .buildkite/release/scripts/generate-nightly-index.py \ --version "$BUILDKITE_COMMIT" \ --current-objects "$obj_json" \ --output-dir "$INDICES_OUTPUT_DIR" \ @@ -78,7 +78,7 @@ if [[ "$version" != *"dev"* ]]; then rm -rf "${INDICES_OUTPUT_DIR:?}" mkdir -p "$INDICES_OUTPUT_DIR" # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path - $PYTHON .buildkite/scripts/generate-nightly-index.py \ + $PYTHON .buildkite/release/scripts/generate-nightly-index.py \ --version "$s3_version" \ --wheel-dir "$BUILDKITE_COMMIT" \ --current-objects "$obj_json" \ diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/release/scripts/generate-nightly-index.py similarity index 100% rename from .buildkite/scripts/generate-nightly-index.py rename to .buildkite/release/scripts/generate-nightly-index.py diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/release/scripts/publish-release-images.sh similarity index 100% rename from .buildkite/scripts/publish-release-images.sh rename to .buildkite/release/scripts/publish-release-images.sh diff --git a/.buildkite/scripts/publish-release-wheels-pypi.sh b/.buildkite/release/scripts/publish-release-wheels-pypi.sh similarity index 100% rename from .buildkite/scripts/publish-release-wheels-pypi.sh rename to .buildkite/release/scripts/publish-release-wheels-pypi.sh diff --git a/.buildkite/scripts/upload-nightly-wheels.sh b/.buildkite/release/scripts/upload-nightly-wheels.sh similarity index 100% rename from .buildkite/scripts/upload-nightly-wheels.sh rename to .buildkite/release/scripts/upload-nightly-wheels.sh diff --git a/.buildkite/scripts/docker_login_ecr_public.sh b/.buildkite/scripts/docker_login_ecr_public.sh deleted file mode 100755 index 51c5e1a5d..000000000 --- a/.buildkite/scripts/docker_login_ecr_public.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Helper function to safely login to ECR Public with per-job config isolation -# Uses DOCKER_CONFIG environment variable to prevent race conditions -# -# This script prevents the "device or resource busy" error by giving each -# Buildkite job its own isolated Docker config directory. -# -# Usage: -# source docker_login_ecr_public.sh && safe_docker_login_ecr_public - -set -euo pipefail - -# Configuration -ECR_REGISTRY="public.ecr.aws" - -setup_isolated_docker_config() { - # Use BUILDKITE_JOB_ID for job-specific isolation - # Fallback to PID if running outside Buildkite - local job_id="${BUILDKITE_JOB_ID:-$$}" - - # Set Docker config to job-specific directory - export DOCKER_CONFIG="/tmp/docker-config-${job_id}" - - # Create directory if it doesn't exist - mkdir -p "$DOCKER_CONFIG" - - echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG" -} - -check_docker_auth() { - # Check if already authenticated to the given registry - # Returns 0 if authenticated, 1 if not - local registry="$1" - - # Check if credentials exist in the isolated config - if [[ -f "$DOCKER_CONFIG/config.json" ]]; then - # Check if registry is present in config - if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then - return 0 - fi - fi - - return 1 -} - -safe_docker_login_ecr_public() { - # Setup isolated config first - setup_isolated_docker_config - - local registry="$ECR_REGISTRY" - - # Check if already authenticated (within this job) - if check_docker_auth "$registry"; then - echo "[docker-login] Already authenticated to $registry in this job" - return 0 - fi - - # Perform login to isolated config directory - echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..." - if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then - echo "[docker-login] Login successful (config: $DOCKER_CONFIG)" - return 0 - else - local exit_code=$? - echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2 - return $exit_code - fi -} - -# Execute if run as script (not sourced) -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - safe_docker_login_ecr_public -fi diff --git a/.buildkite/scripts/hardware_ci/run_npu_test.sh b/.buildkite/scripts/hardware_ci/run_npu_test.sh deleted file mode 100644 index 492699c5f..000000000 --- a/.buildkite/scripts/hardware_ci/run_npu_test.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# This script build the Ascend NPU docker image and run the offline inference inside the container. -# It serves a sanity check for compilation and basic model usage. -set -ex - -# Base ubuntu image with basic ascend development libraries and python installed -VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git" -BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2" -image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" -# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}" -container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" - -# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards -agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}') -echo "agent_idx: ${agent_idx}" -builder_name="cachebuilder${agent_idx}" -builder_cache_dir="/mnt/docker-cache${agent_idx}" -mkdir -p ${builder_cache_dir} - -# Try building the docker image -cat <=6.0 pytest-cov modelscope - -COPY . . - -# Install vllm-omni -WORKDIR /workspace -ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git -ARG VLLM_OMNI_TAG=main -ARG BUILDKITE_PULL_REQUEST -ARG BUILDKITE_PULL_REQUEST_REPO -RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \ - if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \ - echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \ - git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \ - cd /workspace/vllm-omni && \ - git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \ - git checkout pr-\$BUILDKITE_PULL_REQUEST; \ - else \ - echo "Not a PR build, using main branch" && \ - git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \ - fi - -RUN --mount=type=cache,target=/root/.cache/pip \ - export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \ - source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ - source /usr/local/Ascend/nnal/atb/set_env.sh && \ - export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \ - python3 -m pip install -v -e /workspace/vllm-omni/ - -ENV VLLM_WORKER_MULTIPROC_METHOD=spawn - -WORKDIR /workspace/vllm-omni -CMD ["/bin/bash"] - -EOF - -# Setup cleanup -remove_docker_container() { - docker rm -f "${container_name}" || true; - docker image rm -f "${image_name}" || true; - docker system prune -f || true; -} -trap remove_docker_container EXIT - -# Generate corresponding --device args based on BUILDKITE_AGENT_NAME -# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1. -# e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards. -# returns --device /dev/davinci0 --device /dev/davinci1 -parse_and_gen_devices() { - local input="$1" - local index cards_num - if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then - index="${BASH_REMATCH[1]}" - cards_num="${BASH_REMATCH[2]}" - else - echo "parse error" >&2 - return 1 - fi - - local devices="" - local i=0 - while (( i < cards_num )); do - local dev_idx=$(((index - 1)*cards_num + i )) - devices="$devices --device /dev/davinci${dev_idx}" - ((i++)) - done - - # trim leading space - devices="${devices#"${devices%%[![:space:]]*}"}" - # Output devices: assigned to the caller variable - printf '%s' "$devices" -} - -devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1 - -# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware. -# This test checks whether the OOT platform interface is functioning properly in conjunction with -# the hardware plugin vllm-ascend. -hf_model_cache_dir=/mnt/hf_cache${agent_idx} -ms_model_cache_dir=/mnt/modelscope${agent_idx} -mkdir -p ${hf_model_cache_dir} -mkdir -p ${ms_model_cache_dir} -docker run \ - --init \ - ${devices} \ - --device /dev/davinci_manager \ - --device /dev/devmm_svm \ - --device /dev/hisi_hdc \ - -v /usr/local/dcmi:/usr/local/dcmi \ - -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ - -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ - -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ - -v /etc/ascend_install.info:/etc/ascend_install.info \ - -v ${hf_model_cache_dir}:/root/.cache/huggingface \ - -v ${ms_model_cache_dir}:/root/.cache/modelscope \ - --network host \ - --entrypoint="" \ - --name "${container_name}" \ - "${image_name}" \ - bash -c ' - set -e - VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -' diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml deleted file mode 100644 index 520c1bbae..000000000 --- a/.buildkite/test-amd.yaml +++ /dev/null @@ -1,118 +0,0 @@ -steps: - -- label: "Diffusion Model Test" - timeout_in_minutes: 30 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - pytest -s -v tests/e2e/offline_inference/test_z_image.py - -- label: "Diffusion Images API LoRA E2E" - timeout_in_minutes: 30 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py - -- label: "Diffusion Model CPU offloading Test" - timeout_in_minutes: 20 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py - - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py - -- label: "Diffusion Cache Backend Test" - timeout_in_minutes: 15 - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325' - -- label: "Diffusion Sequence Parallelism Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model - -- label: "Diffusion Tensor Parallelism Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py - -- label: "Diffusion GPU Worker Test" - timeout_in_minutes: 20 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - pytest -s -v tests/diffusion/test_diffusion_worker.py - -- label: "Omni Model Test Qwen2-5-Omni" - timeout_in_minutes: 15 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py - - pytest -s -v tests/engine/test_async_omni_engine_abort.py - -- label: "Omni Model Test Qwen3-Omni" - timeout_in_minutes: 15 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_ROCM_USE_AITER=0 - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py - - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py - - -- label: "Omni Sleep Mode Test" - timeout_in_minutes: 40 - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model" diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index dcc2506da..db5068a50 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -9,7 +9,7 @@ description: Generate and run tests for vllm-project/vllm-omni with CI-aligned l Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni). -**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`). +**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`). Default priorities: @@ -96,7 +96,7 @@ Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-styl |----------|------------------|---------------------------|---------------------------| | **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required | | **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` | -| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | +| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/cuda/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | | **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps | | **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) | @@ -460,7 +460,7 @@ When the user asks for **invalid parameter validation**, **invalid request bodie 4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_.py` unless the route is new. -5. **CI — `.buildkite/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`): +5. **CI — `.buildkite/cuda/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`): | Weekly step | Command | When your cases run | |-------------|---------|---------------------| @@ -629,10 +629,10 @@ If the test is not already collected by an existing pipeline command (for exampl | Test level | Edit this file | Typical trigger / intent | |------------|----------------|---------------------------| -| **L1** and **L2** | [`.buildkite/test-ready.yml`](../../../.buildkite/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) | -| **L3** | [`.buildkite/test-merge.yml`](../../../.buildkite/test-merge.yml) | Post-merge; `advanced_model` integration per model type | -| **L4** | [`.buildkite/test-nightly.yml`](../../../.buildkite/test-nightly.yml) | Nightly; grouped by model type (see below) | -| **Invalid param / reliability (weekly)** | [`.buildkite/test-weekly.yml`](../../../.buildkite/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines | +| **L1** and **L2** | [`.buildkite/cuda/test-ready.yml`](../../../.buildkite/cuda/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) | +| **L3** | [`.buildkite/cuda/test-merge.yml`](../../../.buildkite/cuda/test-merge.yml) | Post-merge; `advanced_model` integration per model type | +| **L4** | [`.buildkite/cuda/test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) | Nightly; grouped by model type (see below) | +| **Invalid param / reliability (weekly)** | [`.buildkite/cuda/test-weekly.yml`](../../../.buildkite/cuda/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines | **Level-specific Buildkite delivery (follow the user’s requested level):** @@ -809,21 +809,21 @@ Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.. When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan. -#### L3 / L4: validating Buildkite from a feature branch (root `pipeline.yml`) +#### L3 / L4: validating Buildkite from a feature branch (`cuda/pipeline.yml`) -Production behavior is driven by [`.buildkite/pipeline.yml`](../../../.buildkite/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`). +Production behavior is driven by [`.buildkite/cuda/pipeline.yml`](../../../.buildkite/cuda/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`). When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge): 1. **Point the upload at the right file (optional shortcut)** - In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/test-ready.yml`. For a one-off run you may change that path to `.buildkite/test-merge.yml` (L3) or `.buildkite/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional). + In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/cuda/test-ready.yml`. For a one-off run you may change that path to `.buildkite/cuda/test-merge.yml` (L3) or `.buildkite/cuda/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional). 2. **L3 — merge pipeline** - The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step. + The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `cuda/pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step. 3. **L4 — nightly pipeline** - **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `pipeline.yml` so the nightly definition is uploaded without setting the env var. - Child steps in [`test-nightly.yml`](../../../.buildkite/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload. + **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `cuda/pipeline.yml` so the nightly definition is uploaded without setting the env var. + Child steps in [`test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload. ### Step 6: Run Tests @@ -890,5 +890,5 @@ When completing a request, return: ## Additional Resources - Marker and command routing: [references/test-routing.md](references/test-routing.md) -- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/test-weekly.yml) (invalid param / reliability) +- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/cuda/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/cuda/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/cuda/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/cuda/test-weekly.yml) (invalid param / reliability) - L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910) diff --git a/.claude/skills/vllm-omni-test/references/test-routing.md b/.claude/skills/vllm-omni-test/references/test-routing.md index f78e1c77d..83b9b2007 100644 --- a/.claude/skills/vllm-omni-test/references/test-routing.md +++ b/.claude/skills/vllm-omni-test/references/test-routing.md @@ -2,7 +2,7 @@ Use this reference to map testing goals to levels, markers, and runnable commands. -**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`). +**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`). ## Model-centric e2e filename convention (L2–L4) @@ -224,7 +224,7 @@ When the user’s test plan includes **invalid parameter validation**, **invalid | Sleep / wakeup / server control | `test_invalid_server_control.py` | 3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649). -4. **CI:** [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly. +4. **CI:** [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly. ```bash # Weekly H100 (diffusion / omni / video invalid-param) @@ -291,10 +291,10 @@ When adding or modifying tests, do not stop at “where the file lives” — al | Level | Repo file | Model-type grouping | |-------|-----------|---------------------| -| L1, L2 | [`.buildkite/test-ready.yml`](../../../../.buildkite/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** | -| L3 | [`.buildkite/test-merge.yml`](../../../../.buildkite/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** | -| L4 | [`.buildkite/test-nightly.yml`](../../../../.buildkite/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps | -| Invalid param (weekly) | [`.buildkite/test-weekly.yml`](../../../../.buildkite/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` | +| L1, L2 | [`.buildkite/cuda/test-ready.yml`](../../../../.buildkite/cuda/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** | +| L3 | [`.buildkite/cuda/test-merge.yml`](../../../../.buildkite/cuda/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** | +| L4 | [`.buildkite/cuda/test-nightly.yml`](../../../../.buildkite/cuda/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps | +| Invalid param (weekly) | [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` | ### `source_file_dependencies` (E2E Test only — ready & merge) @@ -380,7 +380,7 @@ Common patterns in `test-ready.yml` / `test-merge.yml`: When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes. -The root [`.buildkite/pipeline.yml`](../../../../.buildkite/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging. +[`.buildkite/cuda/pipeline.yml`](../../../../.buildkite/cuda/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging. Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index e10d8d3b3..63a62eabf 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -131,7 +131,7 @@ Please run the L1 and L2 test cases locally first and attach the results before ### Automatic skip-ci (docs and pytest skip marks) -On pull requests and `main` pushes, the bootstrap step in [`.buildkite/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/pipeline.yml) runs [`.buildkite/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. +On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/cuda/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped. | Change per file | Examples | | --- | --- | diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index c9d86e1c4..d3c164d9b 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -319,7 +319,7 @@ Before entering specific testing levels, the project establishes two common spec ### Diff-aware Buildkite uploads (`source_file_dependencies`) -L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. +L2 (`.buildkite/cuda/test-ready.yml`) and L3 (`.buildkite/cuda/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/cuda/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. #### What `source_file_dependencies` is @@ -334,14 +334,14 @@ L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines | `main` branch push | `git diff --name-only ^..` | | Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped | -Docs-only PRs are handled earlier in bootstrap (`pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. +Docs-only PRs are handled earlier in bootstrap (`.buildkite/cuda/pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. #### Where it is configured | Level | Pipeline file | Upload entry | | --- | --- | --- | -| L2 | `.buildkite/test-ready.yml` | `upload_pipeline.py --upload .buildkite/test-ready.yml` (from `pipeline.yml`) | -| L3 | `.buildkite/test-merge.yml` | `upload_pipeline.py --upload .buildkite/test-merge.yml` (from `pipeline.yml`) | +| L2 | `.buildkite/cuda/test-ready.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-ready.yml` (from `cuda/pipeline.yml`) | +| L3 | `.buildkite/cuda/test-merge.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-merge.yml` (from `cuda/pipeline.yml`) | Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3). @@ -393,10 +393,10 @@ A **group** may also define `source_file_dependencies`; nested steps inherit fil ```bash # Render filtered YAML to stdout (no upload) -python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-ready.yml +python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml # Confirm uploader-only keys are stripped -python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-merge.yml | grep source_file_dependencies +python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies # (no output expected) ``` @@ -404,7 +404,7 @@ On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `s #### Related -- Implementation: `.buildkite/scripts/upload_pipeline.py` +- Implementation: `.buildkite/cuda/scripts/upload_pipeline.py` - L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled. ### Test helper environment variables @@ -871,7 +871,7 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in #### CI trigger -Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: +Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: | Buildkite step | Test file | CI hardware | | -------------- | --------- | ----------- | @@ -902,12 +902,12 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow 1. Add `test_reliability_.py` under `tests/dfx/reliability/`. 2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files). 3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML. -4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/test-weekly.yml`** when the suite should run in weekly L5. +4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/cuda/test-weekly.yml`** when the suite should run in weekly L5. - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/test-weekly.yml` for new models) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) ## Summary diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index ad6a63963..d1179e832 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -49,11 +49,11 @@ Failed jobs: 1/2 cd tests pytest -s -v -m "core_model and cpu" ``` - The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml). + The latest test command is available in the "Simple Unit Test" step of this [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml). === "L2 level" - **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_ready_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_ready_jobs.sh). The script reads [`.buildkite/cuda/test-ready.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-ready.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/ready_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts) for timing and timeout behavior). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All L2 and L1 jobs (default: every pytest step in test-ready.yml) @@ -85,7 +85,7 @@ Failed jobs: 1/2 === "L3 level" - **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned jobs from the repo root via [`tools/run_merge_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/run_merge_jobs.sh). The script reads [`.buildkite/cuda/test-merge.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-merge.yml), generates per-step bash wrappers, runs pytest, and tees logs under `logs/merge_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts)). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All L3 and L1 jobs (default: every pytest step in test-merge.yml) @@ -117,7 +117,7 @@ Failed jobs: 1/2 === "L4 level" - **Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). + **Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/cuda/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). ```bash # All nightly jobs (default: test-type all, model-type all) @@ -189,7 +189,7 @@ Failed jobs: 1/2 ``` - The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml). + The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). You can find more information about markers in the documentation: [marker doc](./tests_markers.md) diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md index 974982b3b..5e5122bfc 100644 --- a/docs/user_guide/fault_injection_reliability_matrix.md +++ b/docs/user_guide/fault_injection_reliability_matrix.md @@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b ## Source of Conclusions -The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`. +The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. ### Example: `SIGTERM` fault injection log (Qwen3-Omni) diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index 7134a36a1..a95eb8876 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-nightly.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-nightly.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # YAML steps whose label contains "Perf Test" run first, then @@ -51,17 +51,17 @@ # Usage: # bash path/to/run_nightly_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash path/to/run_nightly_jobs.sh --test-type perf,acc --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-nightly.yml bash path/to/run_nightly_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-nightly.yml bash path/to/run_nightly_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-nightly.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-nightly.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-nightly.yml exists +# then from the script's directory, until .buildkite/cuda/test-nightly.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/test-nightly.yml) +# YML - path to test-nightly.yml (default: $REPO_ROOT/.buildkite/cuda/test-nightly.yml) # LOG_DIR - logs + generated job scripts; when unset, a timestamped directory under # $REPO_ROOT/logs/ is created: # nightly_jobs_YYYYMMDD-HHMMSS (default / YAML nightly steps) @@ -247,7 +247,7 @@ _nightly_log_dir_basename() { fi } -BUILDKITE_REL=".buildkite/test-nightly.yml" +BUILDKITE_REL=".buildkite/cuda/test-nightly.yml" _find_repo_containing_nightly() { local dir="${1:-}" @@ -267,8 +267,14 @@ _derive_repo_root_from_yml() { local yml="$1" local d d="$(cd "$(dirname "${yml}")" && pwd)" - [[ "$(basename "${d}")" == ".buildkite" ]] || return 1 - printf '%s\n' "$(dirname "${d}")" + while [[ "${d}" != "/" ]]; do + if [[ "$(basename "${d}")" == ".buildkite" ]]; then + printf '%s\n' "$(dirname "${d}")" + return 0 + fi + d="$(dirname "${d}")" + done + return 1 } # Resolve REPO_ROOT, YML (no relative path between script and repo) @@ -278,7 +284,7 @@ if [[ -n "${YML:-}" && -n "${REPO_ROOT:-}" ]]; then elif [[ -n "${YML:-}" ]]; then YML="$(cd "$(dirname "${YML}")" && pwd)/$(basename "${YML}")" if ! REPO_ROOT="$(_derive_repo_root_from_yml "${YML}")"; then - echo "Could not derive REPO_ROOT from YML=${YML} (expected file at /.buildkite/test-nightly.yml)." >&2 + echo "Could not derive REPO_ROOT from YML=${YML} (expected file at /.buildkite/cuda/test-nightly.yml)." >&2 echo "Set REPO_ROOT explicitly (or pass --repo-root) for pytest working directory." >&2 exit 2 fi diff --git a/tools/run_jobs_common.sh b/tools/run_jobs_common.sh index ddc1794dc..7aa0861e4 100644 --- a/tools/run_jobs_common.sh +++ b/tools/run_jobs_common.sh @@ -181,7 +181,7 @@ _run_jobs_print_timing_summary() { # tools/run_merge_jobs.sh). Nightly does not use this entrypoint. # # Entry scripts must set before calling run_yaml_ci_jobs_main: -# BUILDKITE_REL - e.g. .buildkite/test-ready.yml +# BUILDKITE_REL - e.g. .buildkite/cuda/test-ready.yml # DEFAULT_LOG_SUBDIR - e.g. ready_jobs # --------------------------------------------------------------------------- @@ -246,8 +246,14 @@ _run_jobs_derive_repo_root_from_yml() { local yml="$1" local d d="$(cd "$(dirname "${yml}")" && pwd)" - [[ "$(basename "${d}")" == ".buildkite" ]] || return 1 - printf '%s\n' "$(dirname "${d}")" + while [[ "${d}" != "/" ]]; do + if [[ "$(basename "${d}")" == ".buildkite" ]]; then + printf '%s\n' "$(dirname "${d}")" + return 0 + fi + d="$(dirname "${d}")" + done + return 1 } run_yaml_ci_jobs_main() { diff --git a/tools/run_merge_jobs.sh b/tools/run_merge_jobs.sh index 35b38a8e0..9f9ab13b0 100644 --- a/tools/run_merge_jobs.sh +++ b/tools/run_merge_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-merge.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-merge.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # @@ -21,17 +21,17 @@ # Usage: # bash tools/run_merge_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash tools/run_merge_jobs.sh --model-type diffusion --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-merge.yml bash tools/run_merge_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-merge.yml bash tools/run_merge_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-merge.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-merge.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-merge.yml exists +# then from the script's directory, until .buildkite/cuda/test-merge.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/test-merge.yml) +# YML - path to test-merge.yml (default: $REPO_ROOT/.buildkite/cuda/test-merge.yml) # LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/merge_jobs); # per-job *.log plus timing_summary.log after the run # MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above @@ -41,7 +41,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BUILDKITE_REL=".buildkite/test-merge.yml" +BUILDKITE_REL=".buildkite/cuda/test-merge.yml" DEFAULT_LOG_SUBDIR="merge_jobs" # shellcheck source=tools/run_jobs_common.sh diff --git a/tools/run_ready_jobs.sh b/tools/run_ready_jobs.sh index 89f082436..b854faa07 100644 --- a/tools/run_ready_jobs.sh +++ b/tools/run_ready_jobs.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Extract steps from .buildkite/test-ready.yml that contain pytest, synthesize +# Extract steps from .buildkite/cuda/test-ready.yml that contain pytest, synthesize # small bash wrappers (exports + pytest), run them, and tee output to logs named # after each step's Buildkite "key" when present (otherwise a slug of the label). # @@ -21,17 +21,17 @@ # Usage: # bash tools/run_ready_jobs.sh # REPO_ROOT=/path/to/vllm-omni bash tools/run_ready_jobs.sh --model-type omni --dry-run -# YML=/path/to/vllm-omni/.buildkite/test-ready.yml bash tools/run_ready_jobs.sh +# YML=/path/to/vllm-omni/.buildkite/cuda/test-ready.yml bash tools/run_ready_jobs.sh # # Repository / YAML (no dependency on where this script lives): -# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/test-ready.yml +# • Set REPO_ROOT (or pass --repo-root) — default YAML is $REPO_ROOT/.buildkite/cuda/test-ready.yml # • Or set YML (or --yaml) — repo root is inferred as parent of the .buildkite directory # • Or run from inside the clone: git rev-parse --show-toplevel, else walk up from $PWD, -# then from the script's directory, until .buildkite/test-ready.yml exists +# then from the script's directory, until .buildkite/cuda/test-ready.yml exists # # Optional environment: # REPO_ROOT - vllm-omni root (working directory for pytest); see above -# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/test-ready.yml) +# YML - path to test-ready.yml (default: $REPO_ROOT/.buildkite/cuda/test-ready.yml) # LOG_DIR - logs + generated job scripts (default: $REPO_ROOT/logs/ready_jobs); # per-job *.log plus timing_summary.log after the run # MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above @@ -41,7 +41,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BUILDKITE_REL=".buildkite/test-ready.yml" +BUILDKITE_REL=".buildkite/cuda/test-ready.yml" DEFAULT_LOG_SUBDIR="ready_jobs" # shellcheck source=tools/run_jobs_common.sh