env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:

- group: ":card_index_dividers: Simple Test"
  steps:
    - label: "Simple · Model Executor Test · Shard %N/%t"
      agent_pool: mi300_1
      parallelism: 2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml"

    - label: "Simple · Diffusion Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

    - label: "Simple · Engine&Entrypoints Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

    - label: "Simple · Other Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

- label: "Custom Pipeline Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"


- label: "Voxtral TTS CUDA Unit Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py

- group: ":card_index_dividers: Diffusion Test"
  steps:
    - label: "Diffusion · Batch Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion · Offloader Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"

    - label: "Diffusion · Cache Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export GPU_ARCHS=gfx942
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion Sequence Parallelism Test"
      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model

    - label: "Diffusion GPU Worker Test"
      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py

    - label: "Diffusion · Model Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion · Wan22 Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - |
          timeout 40m bash -c "
            export VLLM_TEST_CLEAN_GPU_MEMORY=1
            export VLLM_IMAGE_FETCH_TIMEOUT=60
            pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
          "

## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
# - label: "Audio Generation Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model

- label: "Engine Test"
  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - |
      timeout 15m bash -c '
              pytest -s -v tests/engine/test_async_omni_engine_abort.py
      '

# - label: "Omni Model Test Qwen3-Omni"
#   agent_pool: mi300_2
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
#     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"

- group: ":card_index_dividers: E2E Test"
  steps:
    - label: "Qwen3-TTS E2E Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
        - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"

    - label: "CosyVoice3-TTS E2E Test"
      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - |
          timeout 20m bash -c '
            pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
          '

# TODO: Bagel test on ROCm is very unstable. @tjtanaa
# Need to debug before reneable numerical changes across large PRs
# - label: "Bagel Text2Img Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

# - label: "Bagel Img2Img Model Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

# - label: "Bagel Online Serving Test"
#   agent_pool: mi300_1
#   depends_on: amd-build
#   mirror_hardwares: [amdproduction]
#   grade: Blocking
#   commands:
#     - export GPU_ARCHS=gfx942
#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
#     - export VLLM_LOGGING_LEVEL=DEBUG
#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"