[CI][ROCm] Add non-blocking entrypoint GPU coverage (#7398)
Signed-off-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
1 parent
6040bb97e2
commit
bbf77cce69
1 file changed
+65
@@ -205,6 +205,71 @@ steps:
|
|||||||
pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
||||||
'
|
'
|
||||||
|
|
||||||
|
- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
|
||||||
|
agent_pool: mi300_1
|
||||||
|
depends_on: amd-build
|
||||||
|
mirror_hardwares: [amdproduction]
|
||||||
|
grade: NonBlocking
|
||||||
|
timeout_in_minutes: 30
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
artifact_paths:
|
||||||
|
- "artifacts/rocm-r2-02/**/*"
|
||||||
|
commands:
|
||||||
|
# Native AMD tests run from /app/vllm-omni, while Buildkite uploads
|
||||||
|
# artifacts relative to its checkout directory.
|
||||||
|
- export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
|
||||||
|
- mkdir -p "$$R2_02_ARTIFACT_DIR"
|
||||||
|
- >-
|
||||||
|
ps -eo pid=,ppid=,lstart=,stat=,comm= >
|
||||||
|
"$$R2_02_ARTIFACT_DIR/processes-before.txt"
|
||||||
|
# Verbose pytest output and JUnit retain selected node IDs without a
|
||||||
|
# second collection-only pass over the same GPU-backed test tree.
|
||||||
|
# The inner timeout leaves five minutes for teardown verification and
|
||||||
|
# artifact generation before the Buildkite step timeout.
|
||||||
|
- >-
|
||||||
|
timeout --signal=TERM --kill-after=1m 25m
|
||||||
|
pytest -s -v -ra
|
||||||
|
tests/entrypoints/
|
||||||
|
-m 'core_model and cuda and not
|
||||||
|
(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
|
||||||
|
--run-level "core_model"
|
||||||
|
--durations=0
|
||||||
|
--junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
|
||||||
|
2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
|
||||||
|
- >-
|
||||||
|
tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
|
||||||
|
"$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
|
||||||
|
- |
|
||||||
|
sleep 5
|
||||||
|
ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
|
||||||
|
# Compare PID plus process start time so PID reuse cannot hide a leak.
|
||||||
|
# Double dollars preserve awk field references through Buildkite interpolation.
|
||||||
|
awk '
|
||||||
|
NR == FNR {
|
||||||
|
if ($$9 != "ps") {
|
||||||
|
baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
|
||||||
|
}
|
||||||
|
next
|
||||||
|
}
|
||||||
|
$$9 != "ps" {
|
||||||
|
identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
|
||||||
|
if (!(identity in baseline)) {
|
||||||
|
print
|
||||||
|
leaked = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
END {
|
||||||
|
if (leaked) {
|
||||||
|
print "status=FAIL leaked_processes_detected=1"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
print "status=PASS leaked_processes=0"
|
||||||
|
}
|
||||||
|
' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
|
||||||
|
"$$R2_02_ARTIFACT_DIR/processes-after.txt" \
|
||||||
|
2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
|
||||||
|
|
||||||
# - label: "Omni Model Test Qwen3-Omni"
|
# - label: "Omni Model Test Qwen3-Omni"
|
||||||
# agent_pool: mi300_2
|
# agent_pool: mi300_2
|
||||||
# depends_on: amd-build
|
# depends_on: amd-build
|
||||||
|
|||||||
Reference in new issue
Block a user