diff --git a/.buildkite/common/ci_mirror_hardwares.yml b/.buildkite/common/ci_mirror_hardwares.yml index e3c9a427a..2c58a7c3f 100644 --- a/.buildkite/common/ci_mirror_hardwares.yml +++ b/.buildkite/common/ci_mirror_hardwares.yml @@ -26,6 +26,9 @@ # packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU / # memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm). # Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137). +# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled +# build-level HF_TOKEN values can override pod env entries with the same name; +# upload_pipeline.py restores the secret-backed value inside the job shell. x-npu-k8s-plugins: &npu_k8s_plugins - kubernetes: @@ -84,7 +87,7 @@ mirror_hardwares: env: - name: HF_HOME value: /fsx/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -135,7 +138,7 @@ mirror_hardwares: env: - name: HF_HOME value: /fsx/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -186,7 +189,7 @@ mirror_hardwares: env: - name: HF_HOME value: /fsx/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -237,7 +240,7 @@ mirror_hardwares: env: - name: HF_HOME value: /fsx/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -271,7 +274,7 @@ mirror_hardwares: env: - name: HF_HOME value: /root/.cache/huggingface - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -306,7 +309,7 @@ mirror_hardwares: env: - name: HF_HOME value: /root/.cache/huggingface - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -341,7 +344,7 @@ mirror_hardwares: env: - name: HF_HOME value: /root/.cache/huggingface - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -376,7 +379,7 @@ mirror_hardwares: env: - name: HF_HOME value: /root/.cache/huggingface - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -426,7 +429,7 @@ mirror_hardwares: value: "0" - name: HF_HOME value: /raid/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -477,7 +480,7 @@ mirror_hardwares: value: "0" - name: HF_HOME value: /raid/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -528,7 +531,7 @@ mirror_hardwares: value: "0" - name: HF_HOME value: /raid/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret @@ -579,7 +582,7 @@ mirror_hardwares: value: "0" - name: HF_HOME value: /raid/hf_cache - - name: HF_TOKEN + - name: VLLM_CI_HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index 53da6f660..3772ba59e 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -79,6 +79,8 @@ BOOTSTRAP_UPLOAD_IF_KEYS = { } E2E_GROUP_MARKER = "E2E Test" CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" +CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN" +CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi' # Bootstrap Buildkite ``if`` expressions. # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN). @@ -501,6 +503,17 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None: expanded = copy.deepcopy(preset) merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded + # A scheduled build's HF_TOKEN may override a pod env entry with the same + # name. The preset injects the Kubernetes secret under a collision-proof + # alias; restore the conventional name after Buildkite has merged env. + if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()): + commands = merged.get("commands") + if isinstance(commands, list): + merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands] + elif commands is None: + merged["commands"] = [CUDA_HF_TOKEN_EXPORT] + else: + merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands] # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out. if "retry" in step: merged["retry"] = step["retry"] diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index 9011a1aef..7492a4b82 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "com from skip_ci import resolve_ci_decision # noqa: E402 from upload_pipeline import ( # noqa: E402 + CUDA_HF_TOKEN_EXPORT, _expand_mirror_hardwares, _get_mirror_hw_selector, _load_bootstrap_steps, @@ -143,6 +144,10 @@ def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins(monkeypatch: pytest container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0] assert container["image"].endswith("$BUILDKITE_COMMIT") assert container["resources"]["limits"]["nvidia.com/gpu"] == 1 + env_names = {item["name"] for item in container["env"]} + assert "VLLM_CI_HF_TOKEN" in env_names + assert "HF_TOKEN" not in env_names + assert step["commands"] == [CUDA_HF_TOKEN_EXPORT, "pytest -sv tests/example"] def test_mirror_hardwares_l4_preserves_explicit_retry() -> None: @@ -182,6 +187,33 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None: assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [ {"name": "swr-secret"}, ] + assert step["commands"] == ["pytest -sv tests/example"] + + +def test_all_cuda_mirror_hardwares_restore_hf_token_at_runtime() -> None: + for name in ( + "l4_1", + "l4_2", + "l4_3", + "l4_4", + "h100_1", + "h100_2", + "h100_3", + "h100_4", + "b200_1", + "b200_2", + "b200_3", + "b200_4", + ): + step = _expand_mirror_hardwares( + {"label": name, "mirror_hardwares": name, "commands": ["pytest -sv tests/example"]}, + ) + assert step is not None + container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0] + env_names = {item["name"] for item in container["env"]} + assert "VLLM_CI_HF_TOKEN" in env_names + assert "HF_TOKEN" not in env_names + assert step["commands"][0] == CUDA_HF_TOKEN_EXPORT def _gpu_limit(step: dict) -> int: