diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index 0df78c98e..78c7365cd 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -50,18 +50,9 @@ steps: # that is not supported on CPU # Keep multi-GPU tests out of this one-GPU pool even when their module # also carries the cpu marker for unit tests. + # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run + # the same suite mandatorily, and AMD covers it non-blocking in nightly. - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4, - # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane. - - label: "Simple · Pi0.5 CPU Test" - agent_pool: mi300_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - timeout_in_minutes: 45 - commands: - - export VLLM_ROCM_USE_AITER=0 - - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'" - label: "Diffusion · AuK VAE GPU Test" agent_pool: mi300_1 diff --git a/.buildkite/amd/test-amd-nightly.yml b/.buildkite/amd/test-amd-nightly.yml index 138fb120b..17796f913 100644 --- a/.buildkite/amd/test-amd-nightly.yml +++ b/.buildkite/amd/test-amd-nightly.yml @@ -83,6 +83,20 @@ steps: commands: - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model" + # π0.5 CPU units are mandatory on CUDA READY/MERGE. On AMD they exercise the + # ROCm container/import path with CPU tensors only, so keep them off the + # blocking READY/MERGE critical path and retain non-blocking nightly coverage. + - group: ":robot_face: Pi0.5 CPU Nightly" + steps: + - label: "Simple · Pi0.5 CPU Test" + agent_pool: mi300_1 + mirror_hardwares: [amdproduction] + grade: NonBlocking + timeout_in_minutes: 60 + commands: + - export VLLM_ROCM_USE_AITER=0 + - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'" + - group: ":full_moon: LingBot-Video MI300 Nightly" steps: - label: "LingBot-Video Dense CFG-off Smoke" diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index 628b3b419..9be7ea886 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -50,18 +50,9 @@ steps: # that is not supported on CPU # Keep multi-GPU tests out of this one-GPU pool even when their module # also carries the cpu marker for unit tests. + # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run + # the same suite mandatorily, and AMD covers it non-blocking in nightly. - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - # This module takes about 22 minutes on MI300 and was consistently assigned to shard 4, - # pushing that shard to the 45-minute timeout. Keep its module-scoped fixture in one lane. - - label: "Simple · Pi0.5 CPU Test" - agent_pool: mi300_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - timeout_in_minutes: 45 - commands: - - export VLLM_ROCM_USE_AITER=0 - - "pytest -sv tests/diffusion/models/pi05/test_pi05_units.py -m 'core_model and cpu'" - label: "Diffusion · AuK VAE GPU Test" agent_pool: mi300_1 diff --git a/tests/buildkite/test_pi05_ci_routing.py b/tests/buildkite/test_pi05_ci_routing.py new file mode 100644 index 000000000..14efee63a --- /dev/null +++ b/tests/buildkite/test_pi05_ci_routing.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from pathlib import Path +from shlex import split + +import pytest +import yaml + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml") +AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml") +AMD_READY_PIPELINE = Path(".buildkite/amd/test-amd-ready.yml") +CUDA_MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml") +CUDA_READY_PIPELINE = Path(".buildkite/cuda/test-ready.yml") +PI05_LABEL = "Simple · Pi0.5 CPU Test" +PI05_TEST_PATH = "tests/diffusion/models/pi05/test_pi05_units.py" + + +def _pipeline_steps(pipeline_path: Path) -> list[dict]: + pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) + flattened = [] + + def walk(steps: list[dict]) -> None: + for step in steps: + flattened.append(step) + walk(step.get("steps", [])) + + walk(pipeline.get("steps", [])) + return flattened + + +def _find_step(label: str, pipeline_path: Path) -> dict: + step = next((step for step in _pipeline_steps(pipeline_path) if step.get("label") == label), None) + assert step is not None, f"missing pipeline step: {label}" + return step + + +@pytest.mark.parametrize( + "pipeline_path", + [AMD_READY_PIPELINE, AMD_MERGE_PIPELINE], + ids=["ready", "merge"], +) +def test_pi05_cpu_suite_is_excluded_from_amd_pr_gates(pipeline_path: Path) -> None: + assert all(step.get("label") != PI05_LABEL for step in _pipeline_steps(pipeline_path)) + + step = _find_step("Simple · Diffusion Test · Shard %N/%t", pipeline_path) + pytest_command = next(command for command in step["commands"] if "pytest" in command) + assert f"--ignore={PI05_TEST_PATH}" in split(pytest_command) + + +def test_pi05_cpu_suite_runs_nonblocking_in_amd_nightly() -> None: + steps = [step for step in _pipeline_steps(AMD_NIGHTLY_PIPELINE) if step.get("label") == PI05_LABEL] + assert len(steps) == 1 + + step = steps[0] + assert step["agent_pool"] == "mi300_1" + assert step["grade"] == "NonBlocking" + assert step["timeout_in_minutes"] == 60 + + pytest_command = next(command for command in step["commands"] if "pytest" in command) + argv = split(pytest_command) + assert PI05_TEST_PATH in argv + assert argv[argv.index("-m") + 1] == "core_model and cpu" + + +@pytest.mark.parametrize( + "pipeline_path", + [CUDA_READY_PIPELINE, CUDA_MERGE_PIPELINE], + ids=["ready", "merge"], +) +def test_cuda_pr_gates_retain_pi05_cpu_coverage(pipeline_path: Path) -> None: + step = _find_step("Simple · Diffusion Test", pipeline_path) + pytest_command = next(command for command in step["commands"] if "pytest" in command) + argv = split(pytest_command) + + assert "source_file_dependencies" not in step + assert "tests/diffusion" in argv + assert argv[argv.index("-m") + 1] == "core_model and cpu" + assert f"--ignore={PI05_TEST_PATH}" not in argv