diff --git a/.buildkite/common/ci_mirror_hardwares.yml b/.buildkite/common/ci_mirror_hardwares.yml index e68cbd382..2d4aea7d4 100644 --- a/.buildkite/common/ci_mirror_hardwares.yml +++ b/.buildkite/common/ci_mirror_hardwares.yml @@ -1,19 +1,19 @@ # Uploader-only hardware presets for CUDA / NPU test YAML. # Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to -# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips -# ``mirror_hardwares`` before Buildkite upload. +# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins`` (+ ``retry`` +# for ``l4_*``), then strips ``mirror_hardwares`` before Buildkite upload. # # CUDA naming: {chip}_{count} -# l4_1, l4_4, h100_1 … h100_4 +# l4_1, l4_2, l4_3, l4_4, h100_1 … h100_4 # NPU naming: {queue}_{resource_class} # a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image) # a2b3_npu_4 — ascend-a2b3, resource_class npu-4 # a3_npu_2 — ascend-a3, resource_class npu-2 # -# The ``l4_*`` presets run under the ``docker`` plugin, which does not put -# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes`` -# presets do, so they set ``mount-buildkite-agent: true``; without it any step -# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there. +# The ``l4_*`` presets run on the EKS ``l4-k8s`` queue (agent-stack-k8s), +# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU / +# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm). +# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137). x-npu-k8s-plugins: &npu_k8s_plugins - kubernetes: @@ -21,39 +21,224 @@ x-npu-k8s-plugins: &npu_k8s_plugins imagePullSecrets: - name: swr-secret +# Infra-only retry. Omit exit 1 (pytest failed) and 137 (cgroup OOM / SIGKILL +# of the test process). ci-infra ``K8S_RETRY`` still retries exit 1. +x-l4-k8s-retry: &l4_k8s_retry + automatic: + - exit_status: -1 + limit: 1 + - exit_status: 128 + limit: 1 + - signal_reason: agent_stop + limit: 1 + - signal_reason: agent_refused + limit: 1 + mirror_hardwares: l4_1: agents: - queue: gpu_1_queue + queue: l4-k8s + retry: *l4_k8s_retry plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - mount-buildkite-agent: true - shm-size: "8gb" - environment: - - HF_HOME=/fsx/hf_cache - - HF_TOKEN - volumes: - - /fsx/hf_cache:/fsx/hf_cache + - kubernetes: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + karpenter.sh/do-not-disrupt: "true" + podSpec: + priorityClassName: ci + serviceAccountName: buildkite-gpu-jobs + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + nodeSelector: + vllm.ci/gpu-pool: l4x4 + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + requests: + nvidia.com/gpu: 1 + cpu: "10" + memory: 40Gi + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /fsx/hf_cache + env: + - name: HF_HOME + value: /fsx/hf_cache + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + volumes: + - name: devshm + emptyDir: + medium: Memory + sizeLimit: 24Gi + - name: hf-cache + hostPath: + path: /fsx/hf_cache + type: DirectoryOrCreate + + l4_2: + agents: + queue: l4-k8s + retry: *l4_k8s_retry + plugins: + - kubernetes: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + karpenter.sh/do-not-disrupt: "true" + podSpec: + priorityClassName: ci + serviceAccountName: buildkite-gpu-jobs + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + nodeSelector: + vllm.ci/gpu-pool: l4x4 + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + requests: + nvidia.com/gpu: 2 + cpu: "20" + memory: 80Gi + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /fsx/hf_cache + env: + - name: HF_HOME + value: /fsx/hf_cache + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + volumes: + - name: devshm + emptyDir: + medium: Memory + sizeLimit: 48Gi + - name: hf-cache + hostPath: + path: /fsx/hf_cache + type: DirectoryOrCreate + + l4_3: + agents: + queue: l4-k8s + retry: *l4_k8s_retry + plugins: + - kubernetes: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + karpenter.sh/do-not-disrupt: "true" + podSpec: + priorityClassName: ci + serviceAccountName: buildkite-gpu-jobs + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + nodeSelector: + vllm.ci/gpu-pool: l4x4 + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 3 + requests: + nvidia.com/gpu: 3 + cpu: "30" + memory: 120Gi + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /fsx/hf_cache + env: + - name: HF_HOME + value: /fsx/hf_cache + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + volumes: + - name: devshm + emptyDir: + medium: Memory + sizeLimit: 72Gi + - name: hf-cache + hostPath: + path: /fsx/hf_cache + type: DirectoryOrCreate l4_4: agents: - queue: gpu_4_queue + queue: l4-k8s + retry: *l4_k8s_retry plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - mount-buildkite-agent: true - shm-size: "8gb" - environment: - - HF_HOME=/fsx/hf_cache - - HF_TOKEN - volumes: - - /fsx/hf_cache:/fsx/hf_cache + - kubernetes: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + karpenter.sh/do-not-disrupt: "true" + podSpec: + priorityClassName: ci + serviceAccountName: buildkite-gpu-jobs + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + nodeSelector: + vllm.ci/gpu-pool: l4x4 + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 4 + requests: + nvidia.com/gpu: 4 + cpu: "40" + memory: 160Gi + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /fsx/hf_cache + env: + - name: HF_HOME + value: /fsx/hf_cache + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + volumes: + - name: devshm + emptyDir: + medium: Memory + sizeLimit: 96Gi + - name: hf-cache + hostPath: + path: /fsx/hf_cache + type: DirectoryOrCreate h100_1: agents: diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index a9c0b0561..63e8780c8 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -258,7 +258,11 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]: ) expanded = copy.deepcopy(preset) - return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded + merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded + # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out. + if "retry" in step: + merged["retry"] = step["retry"] + return merged def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool: diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 9f44ba2b0..7c1d5f799 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -13,7 +13,7 @@ steps: steps: - label: ":full_moon: Omni · Function Test with H100 · Single-GPU" - timeout_in_minutes: 90 + timeout_in_minutes: 120 commands: - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py mirror_hardwares: h100_1 @@ -30,11 +30,11 @@ steps: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - - label: ":full_moon: Omni · Doc Test with L4" + - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" + - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model" mirror_hardwares: l4_4 - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU" @@ -178,7 +178,7 @@ steps: build.pull_request.labels includes "nightly-test" steps: - - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)" + - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU" timeout_in_minutes: 30 # NOTE: Single-GPU (core_model) tiny model tests run on the ready label, # so nightly only adds the multi-GPU parallelism configurations. @@ -188,7 +188,14 @@ steps: # for the CI, since these tests should pass irrespective of whether or not we use # the full weights or not. commands: - - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model" + - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model" + mirror_hardwares: l4_2 + + - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU" + timeout_in_minutes: 30 + # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count). + commands: + - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model" mirror_hardwares: l4_4 - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" @@ -220,10 +227,10 @@ steps: --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU" + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU" timeout_in_minutes: 120 commands: - - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy + - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy mirror_hardwares: l4_4 - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" @@ -427,10 +434,22 @@ steps: depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" steps: - - label: ":full_moon: Diffusion · Distributed Test with L4" + - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU" timeout_in_minutes: 60 commands: - - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model" + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model" + mirror_hardwares: l4_2 + + - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU" + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model" + mirror_hardwares: l4_3 + + - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU" + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model" mirror_hardwares: l4_4 - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU" diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 7fc46501b..cb41afbca 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -51,17 +51,20 @@ steps: - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" mirror_hardwares: l4_1 - - label: "Diffusion · Distributed & Attention Test" - source_file_dependencies: - - tests/diffusion/distributed/ - - tests/diffusion/attention/test_attention_sp.py - - vllm_omni/diffusion/distributed/ - - vllm_omni/diffusion/attention/ - - vllm_omni/diffusion/data.py - - vllm_omni/diffusion/models/ + - label: "Diffusion · Distributed Test · Single-GPU" commands: - - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model" - - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model" + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Distributed Test · 2-GPU" + commands: + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model" + mirror_hardwares: l4_2 + + - label: "Diffusion · Distributed & Attention Test · 4-GPU" + commands: + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model" + - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model" mirror_hardwares: l4_4 - label: "Diffusion · Model Test" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 85b5e4c3f..ae1f60971 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -111,17 +111,24 @@ steps: if: build.env("NON_CRITICAL") == "1" steps: - # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job. - - label: "Omni · L4 · Multi-GPU" + # Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card. + - label: "Omni · L4 · 2-GPU" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model" + - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model" + mirror_hardwares: l4_2 + + - label: "Omni · L4 · 4-GPU" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model" - >- pytest -s -v tests/e2e/ --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py - -m "slow and L4 and omni and not cards_1" + -m "slow and L4 and omni and cards_4" --run-level "full_model" mirror_hardwares: l4_4 @@ -194,12 +201,21 @@ steps: --run-level "full_model" mirror_hardwares: l4_1 - - label: "Diffusion · L4 · Multi-GPU" + - label: "Diffusion · L4 · 2-GPU" timeout_in_minutes: 60 commands: - >- pytest -sv - tests/e2e/ -m "slow and diffusion and L4 and not cards_1" + tests/e2e/ -m "slow and diffusion and L4 and cards_2" + --run-level "full_model" + mirror_hardwares: l4_2 + + - label: "Diffusion · L4 · 4-GPU" + timeout_in_minutes: 60 + commands: + - >- + pytest -sv + tests/e2e/ -m "slow and diffusion and L4 and cards_4" --run-level "full_model" mirror_hardwares: l4_4 diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md index a3ffed765..c7cb4409d 100644 --- a/docs/contributing/ci/ci_settings.md +++ b/docs/contributing/ci/ci_settings.md @@ -414,10 +414,12 @@ settings in `pyproject.toml`: online tests launch the server with `subprocess.Popen` and stop it with SIGTERM, and without those settings the XML reflects only the pytest parent process, not the server's code paths. -The upload depends on `buildkite-agent` being callable inside the container. The -`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`) -only do because they set `mount-buildkite-agent: true`. A new docker preset that -runs a coverage job needs the same. +The upload depends on `buildkite-agent` being callable inside the container. +CUDA (`l4_*`, `h100_*`) and NPU (`*_npu_*`) presets all use the `kubernetes` +plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue +(agent-stack-k8s); they no longer use the docker plugin or +`mount-buildkite-agent`. A new **docker** preset that runs a coverage job +still needs `mount-buildkite-agent: true`. List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's `source_file_dependencies` — both change what the job measures, so without them a diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index c994f8dd2..451b553b1 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -129,8 +129,29 @@ def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins() -> None: rendered = _render_test_pipeline(doc, changed_files=None) step = rendered["steps"][0] assert "mirror_hardwares" not in step - assert step["agents"]["queue"] == "gpu_1_queue" - assert step["plugins"][0]["docker#v5.2.0"]["image"].endswith("$BUILDKITE_COMMIT") + assert step["agents"]["queue"] == "l4-k8s" + assert step["retry"] == { + "automatic": [ + {"exit_status": -1, "limit": 1}, + {"exit_status": 128, "limit": 1}, + {"signal_reason": "agent_stop", "limit": 1}, + {"signal_reason": "agent_refused", "limit": 1}, + ], + } + container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0] + assert container["image"].endswith("$BUILDKITE_COMMIT") + assert container["resources"]["limits"]["nvidia.com/gpu"] == 1 + + +def test_mirror_hardwares_l4_preserves_explicit_retry() -> None: + step = _expand_mirror_hardwares( + { + "label": "opt-out", + "mirror_hardwares": "l4_1", + "retry": {"automatic": [{"exit_status": 255, "limit": 2}]}, + }, + ) + assert step["retry"] == {"automatic": [{"exit_status": 255, "limit": 2}]} def test_mirror_hardwares_conflicts_with_explicit_agents() -> None: