vllm-omni/.buildkite/amd/test-amd-ready.yml

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:

- group: ":card_index_dividers: Simple Test"
  steps:
    - label: "Simple · Model Executor Test · Shard %N/%t"
      agent_pool: mi300_1
      parallelism: 3
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      timeout_in_minutes: 90
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        - "pytest -sv tests/model_executor -m 'core_model and cpu and not omni' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"

    - label: "Simple · Omni Processor Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      timeout_in_minutes: 60
      commands:
        - export VLLM_ROCM_USE_AITER=0
        - "pytest -sv tests/model_executor/models/test_omni_processing.py -m 'core_model and cpu and omni'"

    - label: "Simple · Diffusion Test · Shard %N/%t"
      agent_pool: mi300_1
      parallelism: 4
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      timeout_in_minutes: 45
      # Buildkite's status-based rule cannot prove every exit 134 is a GPU hang.
      # It retries 134 once; the Kubernetes job retry starts a fresh pod.
      # Assertions (1), timeouts (124), and OOMs (137) remain non-retryable.
      retry:
        automatic:
          - exit_status: 134
            limit: 1
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        # Keep multi-GPU tests out of this one-GPU pool even when their module
        # also carries the cpu marker for unit tests.
        # Keep π0.5 CPU units out of these shards: CUDA READY/MERGE already run
        # the same suite mandatorily, and AMD covers it non-blocking in nightly.
        - "pytest -sv tests/diffusion -m 'core_model and cpu and not (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)' --ignore=tests/diffusion/cache/test_teacache_extractors.py --ignore=tests/diffusion/models/pi05/test_pi05_units.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"

    - label: "Diffusion · AuK VAE GPU Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      timeout_in_minutes: 40
      commands:
        - export VLLM_ROCM_USE_AITER=0
        - >-
          timeout --signal=TERM --kill-after=2m 35m
          pytest -sv tests/diffusion/models/auk/test_auk_vae_fastpath.py
          -m 'core_model and rocm and cards_1' --run-level core_model

    - label: "Simple · Engine&Entrypoints Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"

    - label: "Simple · Other Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py"

- label: "Custom Pipeline Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"


- label: "Voxtral TTS CUDA Unit Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py

- group: ":card_index_dividers: Diffusion Test"
  steps:
    # Keep real FlashAttention execution out of the CPU-marked Simple lane.
    # AITER may need to build several kernels on a cold worker, so run one
    # pytest worker with job-local build directories and a bounded timeout.
    - label: "ROCm · FlashAttention/AITER GPU Coverage"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: NonBlocking
      timeout_in_minutes: 60
      commands:
        - export VLLM_ROCM_USE_AITER=1
        - export AITER_JIT_DIR="/tmp/vllm-omni-aiter-$$BUILDKITE_JOB_ID"
        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-torch-extensions-$$BUILDKITE_JOB_ID"
        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
        - >-
          timeout --signal=TERM --kill-after=2m 55m
          pytest -s -v -n 1
          tests/diffusion/attention/
          -m 'core_model and rocm and MI325 and cards_1'
          --run-level "core_model"
        # MammothModa2 exercises the same two AITER MHA modules built by the
        # preceding suite. Keep its default-backend run in this job so a cold
        # worker compiles each module once instead of timing out the generic
        # model job.
        - >-
          timeout --signal=TERM --kill-after=2m 10m
          pytest -s -v -n 1
          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
          --run-level "core_model"

    # AR paged attention is real GPU work, including cold AITER compilation.
    # Keep its complete numerical matrix out of the CPU-marked Simple shards.
    - label: "ROCm · AR Diffusion Paged Attention GPU Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      timeout_in_minutes: 20
      commands:
        - export VLLM_ROCM_USE_AITER=1
        - export VLLM_OMNI_AR_FA_REQUIRED=1
        - export AITER_JIT_DIR="/tmp/vllm-omni-ar-aiter-$$BUILDKITE_JOB_ID"
        - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-ar-torch-extensions-$$BUILDKITE_JOB_ID"
        - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
        - >-
          timeout --signal=TERM --kill-after=2m 15m
          pytest -sv tests/diffusion/ar_diffusion/test_paged_attention.py
          -m 'core_model and rocm and MI325 and cards_1' --run-level core_model

    - label: "Diffusion · Batch Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion · Offloader Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"

    - label: "Diffusion · Cache Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export GPU_ARCHS=gfx942
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion Sequence Parallelism Test"
      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
        - >-
          timeout 20m pytest -s -v
          tests/diffusion/attention/test_ulysses_uaa.py::test_ulysses_uaa_2d_mask_layout_matches_baseline
          -m 'core_model and cards_2'
          --run-level "core_model"
        - >-
          timeout 20m pytest -s -v
          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
          -m 'core_model and rocm and cards_2'
          --run-level "core_model"

    - label: "Diffusion GPU Worker Test"
      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py

    - label: "Diffusion · Model Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        # Preserve legacy one-GPU tests without cards_1 while excluding every
        # registered multi-GPU marker from this one-GPU job.
        - >-
          timeout 15m pytest -sv tests/diffusion/models/
          --ignore=tests/diffusion/models/auk/test_auk_vae_fastpath.py
          --ignore=tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
          -m 'core_model and cuda and not
          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
          --run-level "core_model"
        # Keep the model-specific numerical coverage blocking without paying
        # the default ROCm backend's cold AITER compilation cost in this lane.
        - >-
          timeout 10m env DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
          pytest -sv
          tests/diffusion/models/mammoth_moda2/test_dit_attention_cuda.py
          --run-level "core_model"

    - label: "Diffusion · Wan22 Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - |
          timeout 40m bash -c "
            export VLLM_TEST_CLEAN_GPU_MEMORY=1
            export VLLM_IMAGE_FETCH_TIMEOUT=60
            pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
          "

## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
# - label: "Audio Generation Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model

- label: "ROCm · Model Executor GPU Coverage (R2-01)"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: NonBlocking
  timeout_in_minutes: 20
  artifact_paths:
    - "artifacts/rocm-r2-01/**/*"
  commands:
    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
    # artifacts relative to its checkout directory.
    - export R2_01_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-01"
    - mkdir -p "$$R2_01_ARTIFACT_DIR"
    # The dedicated Engine Test below owns the only selected tests/engine case;
    # do not duplicate its model startup in this time-bounded coverage lane.
    # The CUDA ready job also covers tests/worker/test_batched_omni_output.py.
    # Keep the initial ROCm burn-in scoped to tests/model_executor/.
    # Verbose pytest output and JUnit retain node IDs without a second
    # collection-only pass over the same GPU-backed test tree.
    - >-
      pytest -s -v -ra
      tests/model_executor/
      -m 'core_model and cuda and not nvidia_only and not
      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
      --run-level "core_model"
      --durations=0
      --junitxml="$$R2_01_ARTIFACT_DIR/pytest.xml"
      2>&1 | tee "$$R2_01_ARTIFACT_DIR/pytest.log"
    - >-
      tail -n 100 "$$R2_01_ARTIFACT_DIR/pytest.log" >
      "$$R2_01_ARTIFACT_DIR/pytest-summary.txt"

- label: "Engine Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - |
      timeout 15m bash -c '
              pytest -s -v tests/engine/test_async_omni_engine_abort.py
      '

- label: "ROCm · Entrypoints GPU Coverage (R2-02)"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: NonBlocking
  timeout_in_minutes: 30
  env:
    VLLM_WORKER_MULTIPROC_METHOD: spawn
  artifact_paths:
    - "artifacts/rocm-r2-02/**/*"
  commands:
    # Native AMD tests run from /app/vllm-omni, while Buildkite uploads
    # artifacts relative to its checkout directory.
    - export R2_02_ARTIFACT_DIR="$$BUILDKITE_BUILD_CHECKOUT_PATH/artifacts/rocm-r2-02"
    - mkdir -p "$$R2_02_ARTIFACT_DIR"
    - >-
      ps -eo pid=,ppid=,lstart=,stat=,comm= >
      "$$R2_02_ARTIFACT_DIR/processes-before.txt"
    # Verbose pytest output and JUnit retain selected node IDs without a
    # second collection-only pass over the same GPU-backed test tree.
    # The inner timeout leaves five minutes for teardown verification and
    # artifact generation before the Buildkite step timeout.
    - >-
      timeout --signal=TERM --kill-after=1m 25m
      pytest -s -v -ra
      tests/entrypoints/
      -m 'core_model and cuda and not
      (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
      --run-level "core_model"
      --durations=0
      --junitxml="$$R2_02_ARTIFACT_DIR/pytest.xml"
      2>&1 | tee "$$R2_02_ARTIFACT_DIR/pytest.log"
    - >-
      tail -n 100 "$$R2_02_ARTIFACT_DIR/pytest.log" >
      "$$R2_02_ARTIFACT_DIR/pytest-summary.txt"
    - |
      sleep 5
      ps -eo pid=,ppid=,lstart=,stat=,comm= > "$$R2_02_ARTIFACT_DIR/processes-after.txt"
      # Compare PID plus process start time so PID reuse cannot hide a leak.
      # Double dollars preserve awk field references through Buildkite interpolation.
      awk '
        NR == FNR {
          if ($$9 != "ps") {
            baseline[$$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7] = 1
          }
          next
        }
        $$9 != "ps" {
          identity = $$1 "|" $$3 "|" $$4 "|" $$5 "|" $$6 "|" $$7
          if (!(identity in baseline)) {
            print
            leaked = 1
          }
        }
        END {
          if (leaked) {
            print "status=FAIL leaked_processes_detected=1"
            exit 1
          }
          print "status=PASS leaked_processes=0"
        }
      ' "$$R2_02_ARTIFACT_DIR/processes-before.txt" \
        "$$R2_02_ARTIFACT_DIR/processes-after.txt" \
        2>&1 | tee "$$R2_02_ARTIFACT_DIR/process-cleanup.txt"

# - label: "Omni Model Test Qwen3-Omni"
#   agent_pool: mi300_2
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
#     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"

- group: ":card_index_dividers: E2E Test"
  steps:
    - label: "Qwen3-TTS E2E Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
        - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"

    # Keep a fast model/server integration check in the blocking lane. The
    # full default-backend suite remains in the nightly pipeline because a
    # cold ROCm AITER MHA build can add more than 25 minutes to this job.
    - label: "CosyVoice3-TTS E2E Smoke (SDPA)"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      # A ROCm hardware hang aborts the interpreter with 134. Let Buildkite
      # create one fresh pod instead of retrying on the poisoned GPU process.
      retry:
        automatic:
          - exit_status: 134
            limit: 1
      commands:
        - export DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA
        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py::test_voice_clone_zh_002 -m "slow" --run-level "core_model"

# TODO: Bagel test on ROCm is very unstable. @tjtanaa
# Need to debug before reneable numerical changes across large PRs
# - label: "Bagel Text2Img Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

# - label: "Bagel Img2Img Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

# - label: "Bagel Online Serving Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"