From 7e7b99b60fb6f731695256901cebcd59d8d039f1 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 18 Jun 2026 08:25:34 +0000 Subject: [PATCH] [CI][Rebase] Fix Ready/Merge pipeline upload by stripping source_file_dependencies Build #2233 on the vllm-omni-rebase pipeline failed both "Upload Ready Pipeline" and "Upload Merge Pipeline" with: Pipeline upload rejected: `source_file_dependencies` is not a valid property on the `command` step. `source_file_dependencies` is an uploader-only key understood by upload_pipeline.py, not by Buildkite. #4478 switched the rebase pipeline's Ready/Merge steps to raw `buildkite-agent pipeline upload` to run the full e2e suite (bypassing diff-aware skipping), but that also bypassed the key-stripping, so Buildkite rejected the YAML. Add a `--all` flag to upload_pipeline.py that forces the keep-all-steps path (changed_files=None) while still stripping source_file_dependencies, and use it for the rebase pipeline's Ready/Merge (and Nightly) uploads. This restores a valid upload AND retains running the full e2e suite on main builds. Normal-CI .buildkite/pipeline.yml is unchanged (still diff-aware). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: tzhouam --- .buildkite/rebase-pipeline.yaml | 6 +++--- .buildkite/scripts/upload_pipeline.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/rebase-pipeline.yaml index edd98fafd..796ad19a7 100644 --- a/.buildkite/rebase-pipeline.yaml +++ b/.buildkite/rebase-pipeline.yaml @@ -14,7 +14,7 @@ steps: depends_on: image-build key: upload-ready-pipeline commands: - - buildkite-agent pipeline upload .buildkite/test-ready.yml + - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-ready.yml agents: queue: "cpu_queue_premerge" @@ -23,7 +23,7 @@ steps: depends_on: image-build key: upload-merge-pipeline commands: - - buildkite-agent pipeline upload .buildkite/test-merge.yml + - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-merge.yml agents: queue: "cpu_queue_premerge" @@ -33,6 +33,6 @@ steps: key: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" commands: - - buildkite-agent pipeline upload .buildkite/test-nightly.yml + - python3 .buildkite/scripts/upload_pipeline.py --upload --all .buildkite/test-nightly.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/scripts/upload_pipeline.py b/.buildkite/scripts/upload_pipeline.py index 7356144a6..89c6c4975 100644 --- a/.buildkite/scripts/upload_pipeline.py +++ b/.buildkite/scripts/upload_pipeline.py @@ -13,13 +13,16 @@ Test pipeline mode (e.g. test-merge.yml): recognize this uploader-only key). Usage: - python3 upload_pipeline.py [--upload] + python3 upload_pipeline.py [--upload] [--all] # Bootstrap (replaces upload_pipeline_with_skip_ci.sh): python3 upload_pipeline.py --upload .buildkite/pipeline.yml # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py): python3 upload_pipeline.py --upload .buildkite/test-merge.yml + + # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key: + python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml """ from __future__ import annotations @@ -437,11 +440,17 @@ def resolve_pipeline_path(arg: str) -> Path: return ROOT / path -def render_pipeline(path: Path) -> str: +def render_pipeline(path: Path, *, force_all: bool = False) -> str: text = path.read_text(encoding="utf-8") diff_range = resolve_diff_range() changed_files = resolve_changed_files() + # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still + # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds + # run the full e2e suite (see .buildkite/rebase-pipeline.yaml). + if force_all: + changed_files = None + if BOOTSTRAP_MARKER in text: rendered = render_bootstrap_pipeline( text, @@ -480,6 +489,11 @@ def main() -> int: action="store_true", help="Pipe rendered YAML to buildkite-agent pipeline upload", ) + parser.add_argument( + "--all", + action="store_true", + help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies", + ) args = parser.parse_args() path = resolve_pipeline_path(args.pipeline) @@ -487,7 +501,7 @@ def main() -> int: _log(f"missing pipeline file: {path}") return 1 - rendered = render_pipeline(path) + rendered = render_pipeline(path, force_all=args.all) if args.upload: upload_to_buildkite(rendered) else: