diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 181a451bb..505ed1b87 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -205,6 +205,71 @@ steps: pytest -s -v tests/engine/test_async_omni_engine_abort.py ' +- label: "ROCm ยท Entrypoints GPU Coverage (R2-02)" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 30 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + artifact_paths: + - "artifacts/rocm-r2-02/**/*" + commands: + # Native AMD tests run from /app/vllm-omni, while Buildkite uploads + # artifacts relative to its checkout directory. + - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02" + - mkdir -p "$$R2_02_ARTIFACT_DIR" + - >- + ps -eo pid=,ppid=,lstart=,stat=,comm= > + "$$R2_02_ARTIFACT_DIR/processes-before.txt" + # Verbose pytest output and JUnit retain selected node IDs without a + # second collection-only pass over the same GPU-backed test tree. + # The inner timeout leaves five minutes for teardown verification and + # artifact generation before the Buildkite step timeout. + - >- + timeout --signal=TERM --kill-after=1m 25m + pytest -s -v -ra + tests/entrypoints/ + -m 'core_model and cuda and not + (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' + --run-level "core_model" + --durations=0 + --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml" + 2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log" + - >- + tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" > + "$$R2_02_ARTIFACT_DIR/pytest-summary.txt" + - | + sleep 5 + ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt" + # Compare PID plus process start time so PID reuse cannot hide a leak. + # Double dollars preserve awk field references through Buildkite interpolation. + awk ' + NR == FNR { + if ($$9 != "ps") { + baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1 + } + next + } + $$9 != "ps" { + identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7 + if (!(identity in baseline)) { + print + leaked = 1 + } + } + END { + if (leaked) { + print "status=FAIL leaked_processes_detected=1" + exit 1 + } + print "status=PASS leaked_processes=0" + } + ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \ + "$$R2_02_ARTIFACT_DIR/processes-after.txt" \ + 2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt" + # - label: "Omni Model Test Qwen3-Omni" # agent_pool: mi300_2 # depends_on: amd-build