419 lines
16 KiB
YAML
419 lines
16 KiB
YAML
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
|
HF_HUB_ETAG_TIMEOUT: 60
|
|
|
|
steps:
|
|
|
|
- group: ":card_index_dividers: Simple Test"
|
|
steps:
|
|
- label: "Simple · Model Executor Test · Shard %N/%t"
|
|
agent_pool: mi300_1
|
|
parallelism: 3
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
timeout_in_minutes: 90
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
|
# that is not supported on CPU
|
|
- "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
|
|
|
- label: "Simple · Omni Processor Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
timeout_in_minutes: 60
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
- "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"
|
|
|
|
- label: "Simple · Diffusion Test · Shard %N/%t"
|
|
agent_pool: mi300_1
|
|
parallelism: 4
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
timeout_in_minutes: 45
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
|
# that is not supported on CPU
|
|
# Keep multi-GPU tests out of this one-GPU pool even when their module
|
|
# also carries the cpu marker for unit tests.
|
|
- "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
|
# This module takes about 22 minutes on MI300 and was consistently assigned to shard 4,
|
|
# pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane.
|
|
- label: "Simple · Pi0.5 CPU Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
timeout_in_minutes: 45
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
- "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'"
|
|
|
|
- label: "Diffusion · AuK VAE GPU Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
timeout_in_minutes: 40
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
- >-
|
|
timeout --signal=TERM --kill-after=2m 35m
|
|
pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
|
|
-m 'core_model and rocm and cards_1' --run-level core_model
|
|
|
|
- label: "Simple · Engine&Entrypoints Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
- "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
|
|
|
|
- label: "Simple · Other Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=0
|
|
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
|
# that is not supported on CPU
|
|
- "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py"
|
|
|
|
- label: "Custom Pipeline Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
|
|
|
|
|
|
- label: "Voxtral TTS CUDA Unit Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
|
|
|
|
- group: ":card_index_dividers: Diffusion Test"
|
|
steps:
|
|
# Keep real FlashAttention execution out of the CPU-marked Simple lane.
|
|
# AITER may need to build several kernels on a cold worker, so run one
|
|
# pytest worker with job-local build directories and a bounded timeout.
|
|
- label: "ROCm · FlashAttention/AITER GPU Coverage"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: NonBlocking
|
|
timeout_in_minutes: 60
|
|
commands:
|
|
- export VLLM_ROCM_USE_AITER=1
|
|
- export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
|
|
- export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
|
|
- mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
|
|
- >-
|
|
timeout --signal=TERM --kill-after=2m 55m
|
|
pytest -s -v -n 1
|
|
tests/diffusion/attention/
|
|
-m 'core_model and rocm and MI325 and cards_1'
|
|
--run-level "core_model"
|
|
|
|
- label: "Diffusion · Batch Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
|
|
|
- label: "Diffusion · Offloader Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
|
|
|
- label: "Diffusion · Cache Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export GPU_ARCHS=gfx942
|
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
|
|
|
- label: "Diffusion Sequence Parallelism Test"
|
|
agent_pool: mi300_2
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
|
|
- >-
|
|
timeout 20m pytest -s -v
|
|
tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
|
|
-m 'core_model and cards_2'
|
|
--run-level "core_model"
|
|
- >-
|
|
timeout 20m pytest -s -v
|
|
tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
|
|
-m 'core_model and rocm and cards_2'
|
|
--run-level "core_model"
|
|
|
|
- label: "Diffusion GPU Worker Test"
|
|
agent_pool: mi300_2
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
|
|
|
|
- label: "Diffusion · Model Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
# Preserve legacy one-GPU tests without cards_1 while excluding every
|
|
# registered multi-GPU marker from this one-GPU job.
|
|
- >-
|
|
timeout 15m pytest -sv tests/diffusion/models/
|
|
--ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
|
|
-m 'core_model and cuda and not
|
|
(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
|
|
--run-level "core_model"
|
|
|
|
- label: "Diffusion · Wan22 Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- |
|
|
timeout 40m bash -c "
|
|
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
|
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
|
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
|
"
|
|
|
|
## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
|
|
# - label: "Audio Generation Model Test"
|
|
# agent_pool: mi300_1
|
|
# depends_on: amd-build
|
|
# mirror_hardwares: [amdproduction]
|
|
# grade: Blocking
|
|
# commands:
|
|
# - export GPU_ARCHS=gfx942
|
|
# - export VLLM_LOGGING_LEVEL=DEBUG
|
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
|
|
|
|
- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: NonBlocking
|
|
timeout_in_minutes: 20
|
|
artifact_paths:
|
|
- "artifacts/rocm-r2-01/**/*"
|
|
commands:
|
|
# Native AMD tests run from /app/vllm-omni, while Buildkite uploads
|
|
# artifacts relative to its checkout directory.
|
|
- export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
|
|
- mkdir -p "$$R2_01_ARTIFACT_DIR"
|
|
# The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
|
|
# Keep the initial ROCm burn-in scoped to the requested directories.
|
|
# Verbose pytest output and JUnit retain node IDs without a second
|
|
# collection-only pass over the same GPU-backed test tree.
|
|
# The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple
|
|
# AITER kernels on ROCm and is covered by the CUDA ready lane.
|
|
- >-
|
|
pytest -s -v -ra
|
|
tests/engine/ tests/model_executor/
|
|
-m 'core_model and cuda and not
|
|
(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
|
|
-k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics'
|
|
--run-level "core_model"
|
|
--durations=0
|
|
--junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"
|
|
2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log"
|
|
- >-
|
|
tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" >
|
|
"$$R2_01_ARTIFACT_DIR/pytest-summary.txt"
|
|
|
|
- label: "Engine Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- |
|
|
timeout 15m bash -c '
|
|
pytest -s -v tests/engine/test_async_omni_engine_abort.py
|
|
'
|
|
|
|
- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: NonBlocking
|
|
timeout_in_minutes: 30
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
artifact_paths:
|
|
- "artifacts/rocm-r2-02/**/*"
|
|
commands:
|
|
# Native AMD tests run from /app/vllm-omni, while Buildkite uploads
|
|
# artifacts relative to its checkout directory.
|
|
- export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
|
|
- mkdir -p "$$R2_02_ARTIFACT_DIR"
|
|
- >-
|
|
ps -eo pid=,ppid=,lstart=,stat=,comm= >
|
|
"$$R2_02_ARTIFACT_DIR/processes-before.txt"
|
|
# Verbose pytest output and JUnit retain selected node IDs without a
|
|
# second collection-only pass over the same GPU-backed test tree.
|
|
# The inner timeout leaves five minutes for teardown verification and
|
|
# artifact generation before the Buildkite step timeout.
|
|
- >-
|
|
timeout --signal=TERM --kill-after=1m 25m
|
|
pytest -s -v -ra
|
|
tests/entrypoints/
|
|
-m 'core_model and cuda and not
|
|
(cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
|
|
--run-level "core_model"
|
|
--durations=0
|
|
--junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
|
|
2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
|
|
- >-
|
|
tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
|
|
"$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
|
|
- |
|
|
sleep 5
|
|
ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
|
|
# Compare PID plus process start time so PID reuse cannot hide a leak.
|
|
# Double dollars preserve awk field references through Buildkite interpolation.
|
|
awk '
|
|
NR == FNR {
|
|
if ($$9 != "ps") {
|
|
baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
|
|
}
|
|
next
|
|
}
|
|
$$9 != "ps" {
|
|
identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
|
|
if (!(identity in baseline)) {
|
|
print
|
|
leaked = 1
|
|
}
|
|
}
|
|
END {
|
|
if (leaked) {
|
|
print "status=FAIL leaked_processes_detected=1"
|
|
exit 1
|
|
}
|
|
print "status=PASS leaked_processes=0"
|
|
}
|
|
' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
|
|
"$$R2_02_ARTIFACT_DIR/processes-after.txt" \
|
|
2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"
|
|
|
|
# - label: "Omni Model Test Qwen3-Omni"
|
|
# agent_pool: mi300_2
|
|
# depends_on: amd-build
|
|
# mirror_hardwares: [amdproduction]
|
|
# grade: Blocking
|
|
# commands:
|
|
# - export VLLM_LOGGING_LEVEL=DEBUG
|
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
|
|
# - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
|
|
|
|
- group: ":card_index_dividers: E2E Test"
|
|
steps:
|
|
- label: "Qwen3-TTS E2E Test"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
commands:
|
|
- export VLLM_LOGGING_LEVEL=DEBUG
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
|
|
|
|
# Keep a fast model/server integration check in the blocking lane. The
|
|
# full default-backend suite remains in the nightly pipeline because a
|
|
# cold ROCm AITER MHA build can add more than 25 minutes to this job.
|
|
- label: "CosyVoice3-TTS E2E Smoke (SDPA)"
|
|
agent_pool: mi300_1
|
|
depends_on: amd-build
|
|
mirror_hardwares: [amdproduction]
|
|
grade: Blocking
|
|
# A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
|
|
# create one fresh pod instead of retrying on the poisoned GPU process.
|
|
retry:
|
|
automatic:
|
|
- exit_status: 134
|
|
limit: 1
|
|
commands:
|
|
- export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
|
|
- pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002 -m "slow" --run-level "core_model"
|
|
|
|
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
|
# Need to debug before reneable numerical changes across large PRs
|
|
# - label: "Bagel Text2Img Model Test"
|
|
# agent_pool: mi300_1
|
|
# depends_on: amd-build
|
|
# mirror_hardwares: [amdproduction]
|
|
# grade: Blocking
|
|
# commands:
|
|
# - export GPU_ARCHS=gfx942
|
|
# - export VLLM_LOGGING_LEVEL=DEBUG
|
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
|
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|
|
|
|
# - label: "Bagel Img2Img Model Test"
|
|
# agent_pool: mi300_1
|
|
# depends_on: amd-build
|
|
# mirror_hardwares: [amdproduction]
|
|
# grade: Blocking
|
|
# commands:
|
|
# - export GPU_ARCHS=gfx942
|
|
# - export VLLM_LOGGING_LEVEL=DEBUG
|
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
|
# - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|
|
|
|
# - label: "Bagel Online Serving Test"
|
|
# agent_pool: mi300_1
|
|
# depends_on: amd-build
|
|
# mirror_hardwares: [amdproduction]
|
|
# grade: Blocking
|
|
# commands:
|
|
# - export GPU_ARCHS=gfx942
|
|
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
|
|
# - export VLLM_LOGGING_LEVEL=DEBUG
|
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
|
# - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
|