anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI] Preserve secret-backed HF token in scheduled GPU jobs (#7118)

Browse Source 
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk>
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
Zhou Taichang authored and tzhouam committed 2026-09-10 00:41:53 +08:00
1 parent 9af0331b21
commit 90b7b6c5c9
3 files changed
+60 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/common/ci_mirror_hardwares.yml
+15 -12
View File
@@ -26,6 +26,9 @@
# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled
# build-level HF_TOKEN values can override pod env entries with the same name;
# upload_pipeline.py restores the secret-backed value inside the job shell.
x-npu-k8s-plugins: &npu_k8s_plugins
- kubernetes:
@@ -84,7 +87,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /fsx/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -135,7 +138,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /fsx/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -186,7 +189,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /fsx/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -237,7 +240,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /fsx/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -271,7 +274,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -306,7 +309,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -341,7 +344,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -376,7 +379,7 @@ mirror_hardwares:
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -426,7 +429,7 @@ mirror_hardwares:
value: "0"
- name: HF_HOME
value: /raid/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -477,7 +480,7 @@ mirror_hardwares:
value: "0"
- name: HF_HOME
value: /raid/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -528,7 +531,7 @@ mirror_hardwares:
value: "0"
- name: HF_HOME
value: /raid/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
@@ -579,7 +582,7 @@ mirror_hardwares:
value: "0"
- name: HF_HOME
value: /raid/hf_cache
- name: HF_TOKEN
- name: VLLM_CI_HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
.buildkite/common/scripts/upload_pipeline.py
+13
View File
@@ -79,6 +79,8 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
}
E2E_GROUP_MARKER = "E2E Test"
CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
# Bootstrap Buildkite ``if`` expressions.
# ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -501,6 +503,17 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
expanded = copy.deepcopy(preset)
merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
# A scheduled build's HF_TOKEN may override a pod env entry with the same
# name. The preset injects the Kubernetes secret under a collision-proof
# alias; restore the conventional name after Buildkite has merged env.
if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
commands = merged.get("commands")
if isinstance(commands, list):
merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
elif commands is None:
merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
else:
merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
# Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
if "retry" in step:
merged["retry"] = step["retry"]
tests/buildkite/test_upload_pipeline.py
+32
View File
@@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "com
from skip_ci import resolve_ci_decision # noqa: E402
from upload_pipeline import ( # noqa: E402
CUDA_HF_TOKEN_EXPORT,
_expand_mirror_hardwares,
_get_mirror_hw_selector,
_load_bootstrap_steps,
@@ -143,6 +144,10 @@ def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins(monkeypatch: pytest
container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0]
assert container["image"].endswith("$BUILDKITE_COMMIT")
assert container["resources"]["limits"]["nvidia.com/gpu"] == 1
env_names = {item["name"] for item in container["env"]}
assert "VLLM_CI_HF_TOKEN" in env_names
assert "HF_TOKEN" not in env_names
assert step["commands"] == [CUDA_HF_TOKEN_EXPORT, "pytest -sv tests/example"]
def test_mirror_hardwares_l4_preserves_explicit_retry() -> None:
@@ -182,6 +187,33 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None:
assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [
{"name": "swr-secret"},
]
assert step["commands"] == ["pytest -sv tests/example"]
def test_all_cuda_mirror_hardwares_restore_hf_token_at_runtime() -> None:
for name in (
"l4_1",
"l4_2",
"l4_3",
"l4_4",
"h100_1",
"h100_2",
"h100_3",
"h100_4",
"b200_1",
"b200_2",
"b200_3",
"b200_4",
):
step = _expand_mirror_hardwares(
{"label": name, "mirror_hardwares": name, "commands": ["pytest -sv tests/example"]},
)
assert step is not None
container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0]
env_names = {item["name"] for item in container["env"]}
assert "VLLM_CI_HF_TOKEN" in env_names
assert "HF_TOKEN" not in env_names
assert step["commands"][0] == CUDA_HF_TOKEN_EXPORT
def _gpu_limit(step: dict) -> int: