[CI] Migrate L4 jobs to l4-k8s and split by GPU count (#6890)

Signed-off-by: wangyu <410167048@qq.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

.buildkite/common/ci_mirror_hardwares.yml

@@ -1,19 +1,19 @@
 # Uploader-only hardware presets for CUDA / NPU test YAML.
 # Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
-# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
-# ``mirror_hardwares`` before Buildkite upload.
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins`` (+ ``retry``
+# for ``l4_*``), then strips ``mirror_hardwares`` before Buildkite upload.
 #
 # CUDA naming: {chip}_{count}
-#   l4_1, l4_4, h100_1 … h100_4
+#   l4_1, l4_2, l4_3, l4_4, h100_1 … h100_4
 # NPU naming: {queue}_{resource_class}
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2
 #
-# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
-# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
-# presets do, so they set ``mount-buildkite-agent: true``; without it any step
-# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.
+# The ``l4_*`` presets run on the EKS ``l4-k8s`` queue (agent-stack-k8s),
+# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
+# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
+# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -21,39 +21,224 @@ x-npu-k8s-plugins: &npu_k8s_plugins
         imagePullSecrets:
           - name: swr-secret
 
+# Infra-only retry. Omit exit 1 (pytest failed) and 137 (cgroup OOM / SIGKILL
+# of the test process). ci-infra ``K8S_RETRY`` still retries exit 1.
+x-l4-k8s-retry: &l4_k8s_retry
+  automatic:
+    - exit_status: -1
+      limit: 1
+    - exit_status: 128
+      limit: 1
+    - signal_reason: agent_stop
+      limit: 1
+    - signal_reason: agent_refused
+      limit: 1
+
 
 mirror_hardwares:
   l4_1:
     agents:
-      queue: gpu_1_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                  requests:
+                    nvidia.com/gpu: 1
+                    cpu: "10"
+                    memory: 40Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 24Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_2:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                  requests:
+                    nvidia.com/gpu: 2
+                    cpu: "20"
+                    memory: 80Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 48Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_3:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                  requests:
+                    nvidia.com/gpu: 3
+                    cpu: "30"
+                    memory: 120Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 72Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   l4_4:
     agents:
-      queue: gpu_4_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                  requests:
+                    nvidia.com/gpu: 4
+                    cpu: "40"
+                    memory: 160Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 96Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   h100_1:
     agents:

.buildkite/common/scripts/upload_pipeline.py

@@ -258,7 +258,11 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
         )
 
     expanded = copy.deepcopy(preset)
-    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
+    if "retry" in step:
+        merged["retry"] = step["retry"]
+    return merged
 
 
 def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:

.buildkite/cuda/test-nightly.yml

@@ -13,7 +13,7 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
-        timeout_in_minutes: 90
+        timeout_in_minutes: 120
         commands:
           - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
         mirror_hardwares: h100_1
@@ -30,11 +30,11 @@ steps:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: ":full_moon: Omni · Doc Test with L4"
+      - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
@@ -178,7 +178,7 @@ steps:
       build.pull_request.labels includes "nightly-test"
     steps:
 
-      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -188,7 +188,14 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        timeout_in_minutes: 30
+        # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
+        commands:
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
@@ -220,10 +227,10 @@ steps:
             --run-level "full_model"
         mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -427,10 +434,22 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion · Distributed Test with L4"
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
+        mirror_hardwares: l4_3
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"

.buildkite/cuda/test-ready.yml

@@ -51,17 +51,20 @@ steps:
           - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · Distributed & Attention Test"
-        source_file_dependencies:
-          - tests/diffusion/distributed/
-          - tests/diffusion/attention/test_attention_sp.py
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/diffusion/attention/
-          - vllm_omni/diffusion/data.py
-          - vllm_omni/diffusion/models/
+      - label: "Diffusion · Distributed Test · Single-GPU"
         commands:
-          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Distributed Test · 2-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
       - label: "Diffusion · Model Test"

.buildkite/cuda/test-weekly.yml

@@ -111,17 +111,24 @@ steps:
     if: build.env("NON_CRITICAL") == "1"
     steps:
 
-      # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
-      - label: "Omni · L4 · Multi-GPU"
+      # Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card.
+      - label: "Omni · L4 · 2-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
+          - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Omni · L4 · 4-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model"
           - >-
             pytest -s -v tests/e2e/
             --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
             --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-            -m "slow and L4 and omni and not cards_1"
+            -m "slow and L4 and omni and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 
@@ -194,12 +201,21 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · L4 · Multi-GPU"
+      - label: "Diffusion · L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
+            tests/e2e/ -m "slow and diffusion and L4 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4 and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 

docs/contributing/ci/ci_settings.md

@@ -414,10 +414,12 @@ settings in `pyproject.toml`: online tests launch the server with
 `subprocess.Popen` and stop it with SIGTERM, and without those settings the XML
 reflects only the pytest parent process, not the server's code paths.
 
-The upload depends on `buildkite-agent` being callable inside the container. The
-`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`)
-only do because they set `mount-buildkite-agent: true`. A new docker preset that
-runs a coverage job needs the same.
+The upload depends on `buildkite-agent` being callable inside the container.
+CUDA (`l4_*`, `h100_*`) and NPU (`*_npu_*`) presets all use the `kubernetes`
+plugin, which provides the agent. `l4_*` jobs run on the EKS `l4-k8s` queue
+(agent-stack-k8s); they no longer use the docker plugin or
+`mount-buildkite-agent`. A new **docker** preset that runs a coverage job
+still needs `mount-buildkite-agent: true`.
 
 List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
 `source_file_dependencies` — both change what the job measures, so without them a

tests/buildkite/test_upload_pipeline.py

@@ -129,8 +129,29 @@ def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins() -> None:
     rendered = _render_test_pipeline(doc, changed_files=None)
     step = rendered["steps"][0]
     assert "mirror_hardwares" not in step
-    assert step["agents"]["queue"] == "gpu_1_queue"
-    assert step["plugins"][0]["docker#v5.2.0"]["image"].endswith("$BUILDKITE_COMMIT")
+    assert step["agents"]["queue"] == "l4-k8s"
+    assert step["retry"] == {
+        "automatic": [
+            {"exit_status": -1, "limit": 1},
+            {"exit_status": 128, "limit": 1},
+            {"signal_reason": "agent_stop", "limit": 1},
+            {"signal_reason": "agent_refused", "limit": 1},
+        ],
+    }
+    container = step["plugins"][0]["kubernetes"]["podSpec"]["containers"][0]
+    assert container["image"].endswith("$BUILDKITE_COMMIT")
+    assert container["resources"]["limits"]["nvidia.com/gpu"] == 1
+
+
+def test_mirror_hardwares_l4_preserves_explicit_retry() -> None:
+    step = _expand_mirror_hardwares(
+        {
+            "label": "opt-out",
+            "mirror_hardwares": "l4_1",
+            "retry": {"automatic": [{"exit_status": 255, "limit": 2}]},
+        },
+    )
+    assert step["retry"] == {"automatic": [{"exit_status": 255, "limit": 2}]}
 
 
 def test_mirror_hardwares_conflicts_with_explicit_agents() -> None: