vllm-omni/.buildkite/cuda/test-weekly.yml

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  VLLM_USE_FLASHINFER_MOE_FP16: 0
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:

  - label: "Reliability Test - qwen3-omni"
    timeout_in_minutes: 180
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    commands:
      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
    mirror_hardwares: h100_2

  - label: "Reliability Test - wan22"
    timeout_in_minutes: 180
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    commands:
      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
    mirror_hardwares: h100_2

  - label: "Reliability Test - hunyuan-image"
    timeout_in_minutes: 180
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    commands:
      - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
    mirror_hardwares: h100_4

  - label: "Reliability Test - voxcpm2"
    timeout_in_minutes: 180
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    commands:
      - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
    mirror_hardwares: l4_1

  - group: "Reliability Test - Invalid parameters Test"
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    steps:

      - label: "Invalid parameters Test · H100"
        timeout_in_minutes: 120
        commands:
          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
        mirror_hardwares: h100_2

      - label: "Invalid parameters Test · L4"
        timeout_in_minutes: 120
        commands:
          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: E2E Tests"
    depends_on: upload-weekly-pipeline
    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
    steps:

      - label: "Omni · L4"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
          - >-
            pytest -s -v tests/e2e/
            --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
            --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
            -m "slow and L4 and omni"
            --run-level "full_model"
        mirror_hardwares: l4_4

      - label: "Omni · H100"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and H100 and omni"
            --run-level "full_model"
        mirror_hardwares: h100_2

      - label: "TTS · L4"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - export VLLM_USE_DEEP_GEMM="0"
          - export VLLM_MOE_USE_DEEP_GEMM="0"
          - |
            failed=0
            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
            done
            exit $$failed
        mirror_hardwares: l4_1

      - label: "TTS · H100"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - export VLLM_USE_DEEP_GEMM="0"
          - export VLLM_MOE_USE_DEEP_GEMM="0"
          - >-
            pytest -sv
            tests/e2e/ -m "slow and H100 and tts"
            --run-level "full_model"
        mirror_hardwares: h100_1

      # Own step: the image has no NanoCodec, so this file import-skips out of
      # the mark-selected TTS steps above. The overrides keep nemo_toolkit[tts]
      # from downgrading transformers; see
      # examples/offline_inference/text_to_speech/README.md.
      - label: "TTS · Gepard-1.0 · L4"
        timeout_in_minutes: 60
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
          - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
          - >-
            pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
            -m "tts and cuda"
            --run-level "full_model"
        mirror_hardwares: l4_1

      - label: "Diffusion · L4"
        timeout_in_minutes: 60
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and diffusion and L4"
            --run-level "full_model"
        mirror_hardwares: l4_4

      - label: "Diffusion · H100 · Single-GPU"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
            --run-level "full_model"
        mirror_hardwares: h100_1

      - label: "Diffusion · H100 · Multi-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
            --run-level "full_model"
        mirror_hardwares: h100_2