vllm-omni/.buildkite/test-nightly.yml

steps:
  - label: ":full_moon: Omni Model Test with H100"
    timeout_in_minutes: 90
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1"
    commands:
      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
      - |
        set +e
        pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "advanced_model" --run-level "advanced_model"
        EXIT1=$$?
        pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
        EXIT2=$$?
        exit $$((EXIT1 | EXIT2))
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 2
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: ":full_moon: Omni Model Test"
    timeout_in_minutes: 60
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1"
    commands:
      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
      - pytest -s -v tests/examples/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
    agents:
      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: ":full_moon: Diffusion Model Test with H100"
    timeout_in_minutes: 60
    depends_on: upload-nightly-pipeline
    # if: build.env("NIGHTLY") == "1"
    commands:
      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
      - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and diffusion and H100" --run-level "advanced_model"
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 2
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: ":full_moon: Qwen3-TTS Non-Async-Chunk E2E Test"
    timeout_in_minutes: 30
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1"
    commands:
      - |
        huggingface-cli download Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice
        export VLLM_WORKER_MULTIPROC_METHOD=spawn
        pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py::TestQwen3TTSNoAsyncChunk
    agents:
      queue: "gpu_4_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: ":full_moon: Omni Model Perf Test & Test Case Statistics"
    key: nightly-performance
    timeout_in_minutes: 180
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1"
    commands:
      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
      - export BENCHMARK_DIR=tests
      - pytest -s -v tests/perf/scripts/run_benchmark.py
      - buildkite-agent artifact upload "tests/*.json"
      - python tools/nightly/buildkite_testcase_statistics.py -o tests/buildkite_testcase_statistics.html
      - buildkite-agent artifact upload "tests/*.html"
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 2
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: ":email: Nightly Collection & Email"
    key: nightly-perf-distribution
    depends_on:
    - nightly-performance
    if: build.env("NIGHTLY") == "1"
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests
      - export DEFAULT_OUTPUT_DIR=tests
      - buildkite-agent artifact download "tests/*.json" . --step nightly-performance
      - buildkite-agent artifact download "tests/*.html" . --step nightly-performance
      - python tools/nightly/generate_nightly_perf_excel.py
      - python tools/nightly/send_nightly_email.py --report-file "tests/*.xlsx, tests/*.html"
      - buildkite-agent artifact upload "tests/*.xlsx"
    agents:
      queue: "cpu_queue_premerge"