diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 585f089a2..181a451bb 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -160,6 +160,39 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model +- label: "ROCm · Engine & Model Executor GPU Coverage (R2-01)" + agent_pool: mi300_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 20 + artifact_paths: + - "artifacts/rocm-r2-01/**/*" + commands: + # Native AMD tests run from /app/vllm-omni, while Buildkite uploads + # artifacts relative to its checkout directory. + - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01" + - mkdir -p "$$R2_01_ARTIFACT_DIR" + # The CUDA ready job also covers tests/worker/test_batched_omni_output.py. + # Keep the initial ROCm burn-in scoped to the requested directories. + # Verbose pytest output and JUnit retain node IDs without a second + # collection-only pass over the same GPU-backed test tree. + # The CUDA/L4-only CosyVoice3 ragged-flow parity test cold-JITs multiple + # AITER kernels on ROCm and is covered by the CUDA ready lane. + - >- + pytest -s -v -ra + tests/engine/ tests/model_executor/ + -m 'core_model and cuda and not + (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' + -k 'not test_code2wav_streaming_batch_matches_ragged_flow_numerics' + --run-level "core_model" + --durations=0 + --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml" + 2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log" + - >- + tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" > + "$$R2_01_ARTIFACT_DIR/pytest-summary.txt" + - label: "Engine Test" agent_pool: mi300_1 depends_on: amd-build diff --git a/tests/buildkite/test_amd_ready_r2_01.py b/tests/buildkite/test_amd_ready_r2_01.py new file mode 100644 index 000000000..9d5147982 --- /dev/null +++ b/tests/buildkite/test_amd_ready_r2_01.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from pathlib import Path +from shlex import split + +import pytest +import yaml + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml") +R2_01_LABEL = "ROCm · Engine & Model Executor GPU Coverage (R2-01)" +SINGLE_GPU_MARKERS = ( + "core_model and cuda and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)" +) +EXCLUDED_TEST = "test_code2wav_streaming_batch_matches_ragged_flow_numerics" + + +def _find_step(label: str) -> dict: + pipeline = yaml.safe_load(AMD_READY_PIPELINE.read_text(encoding="utf-8")) + + def walk(steps: list[dict]) -> dict | None: + for step in steps: + if step.get("label") == label: + return step + if nested := walk(step.get("steps", [])): + return nested + return None + + step = walk(pipeline.get("steps", [])) + assert step is not None, f"missing AMD pipeline step: {label}" + return step + + +def _assert_single_gpu_selection(command: str) -> None: + argv = split(command) + assert "tests/engine/" in argv + assert "tests/model_executor/" in argv + assert "tests/worker/test_batched_omni_output.py" not in argv + assert argv[argv.index("-m") + 1] == SINGLE_GPU_MARKERS + assert argv[argv.index("-k") + 1] == f"not {EXCLUDED_TEST}" + assert argv[argv.index("--run-level") + 1] == "core_model" + + +def test_r2_01_is_nonblocking_single_gpu_coverage() -> None: + step = _find_step(R2_01_LABEL) + + assert step["agent_pool"] == "mi300_1" + assert step["grade"] == "NonBlocking" + assert step["timeout_in_minutes"] == 20 + assert step["mirror_hardwares"] == ["amdproduction"] + assert step["artifact_paths"] == ["artifacts/rocm-r2-01/**/*"] + + +def test_r2_01_runs_once_fails_closed_and_publishes_results() -> None: + commands = _find_step(R2_01_LABEL)["commands"] + pytest_commands = [command for command in commands if command.lstrip().startswith("pytest ")] + assert len(pytest_commands) == 1 + run_command = pytest_commands[0] + artifact_dir = "$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01" + + _assert_single_gpu_selection(run_command) + assert "--collect-only" not in "\n".join(commands) + assert "VLLM_CI_ALLOW_NO_TESTS" not in "\n".join(commands) + assert any(artifact_dir in command for command in commands) + assert "-v" in split(run_command) + assert "-ra" in split(run_command) + assert "--durations=0" in split(run_command) + assert "--junitxml=$$R2_01_ARTIFACT_DIR/pytest.xml" in split(run_command) + assert "$$R2_01_ARTIFACT_DIR/pytest.log" in run_command + assert any("$$R2_01_ARTIFACT_DIR/pytest-summary.txt" in command for command in commands)