[CI][ROCm] Add non-blocking entrypoint GPU coverage (#7398)
Signed-off-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
1 parent
6040bb97e2
commit
bbf77cce69
1 file changed
+65
@@ -205,6 +205,71 @@ steps:
|
||||
pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
||||
'
|
||||
|
||||
- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: NonBlocking
|
||||
timeout_in_minutes: 30
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
artifact_paths:
|
||||
- "artifacts/rocm-r2-02/**/*"
|
||||
commands:
|
||||
# Native AMD tests run from /app/vllm-omni, while Buildkite uploads
|
||||
# artifacts relative to its checkout directory.
|
||||
- export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
|
||||
- mkdir -p "$$R2_02_ARTIFACT_DIR"
|
||||
- >-
|
||||
ps -eo pid=,ppid=,lstart=,stat=,comm= >
|
||||
"$$R2_02_ARTIFACT_DIR/processes-before.txt"
|
||||
# Verbose pytest output and JUnit retain selected node IDs without a
|
||||
# second collection-only pass over the same GPU-backed test tree.
|
||||
# The inner timeout leaves five minutes for teardown verification and
|
||||
# artifact generation before the Buildkite step timeout.
|
||||
- >-
|
||||
timeout --signal=TERM --kill-after=1m 25m
|
||||
pytest -s -v -ra
|
||||
tests/entrypoints/
|
||||
-m 'core_model and cuda and not
|
||||
(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
|
||||
--run-level "core_model"
|
||||
--durations=0
|
||||
--junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
|
||||
2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
|
||||
- >-
|
||||
tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
|
||||
"$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
|
||||
- |
|
||||
sleep 5
|
||||
ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
|
||||
# Compare PID plus process start time so PID reuse cannot hide a leak.
|
||||
# Double dollars preserve awk field references through Buildkite interpolation.
|
||||
awk '
|
||||
NR == FNR {
|
||||
if ($$9 != "ps") {
|
||||
baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
|
||||
}
|
||||
next
|
||||
}
|
||||
$$9 != "ps" {
|
||||
identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
|
||||
if (!(identity in baseline)) {
|
||||
print
|
||||
leaked = 1
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (leaked) {
|
||||
print "status=FAIL leaked_processes_detected=1"
|
||||
exit 1
|
||||
}
|
||||
print "status=PASS leaked_processes=0"
|
||||
}
|
||||
' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
|
||||
"$$R2_02_ARTIFACT_DIR/processes-after.txt" \
|
||||
2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
|
||||
|
||||
# - label: "Omni Model Test Qwen3-Omni"
|
||||
# agent_pool: mi300_2
|
||||
# depends_on: amd-build
|
||||
|
||||
Reference in new issue
Block a user