vllm-omni/.buildkite/test-ready.yml

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  - label: "Simple Unit Test"
    depends_on: upload-ready-pipeline
    commands:
      - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "CUDA Unit Test with single card"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 10m pytest -v -s -m 'core_model and cuda and L4 and not distributed_cuda' --ignore=tests/e2e --ignore=tests/engine/test_async_omni_engine_abort.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "CUDA Unit Test with multi cards"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 10m pytest -v -s -m 'core_model and cuda and L4 and distributed_cuda' --ignore=tests/e2e --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml
    agents:
      queue: "gpu_4_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Diffusion Model Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 30m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Diffusion Batching Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
    agents:
      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Custom Pipeline Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Diffusion Model CPU offloading Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 10m pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
    agents:
      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "AudioX Online Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 20m pytest -s -v tests/e2e/online_serving/test_audiox_online.py -m "core_model and diffusion" --run-level core_model
    agents:
      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Diffusion Cache Backend Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 15m pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4'
    agents:
      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Diffusion Sequence Parallelism Test"
    depends_on: upload-ready-pipeline
    commands:
      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
    agents:
      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"


  - label: "Engine Test"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 15m bash -c '
                pytest -s -v tests/engine/test_async_omni_engine_abort.py
        '
    agents:
      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Omni Model Test with H100"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 2
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: "MiMo-Audio E2E Test with H100"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 30m bash -c '
          export VLLM_LOGGING_LEVEL=DEBUG
          pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: "Qwen3-TTS E2E Test"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          export VLLM_LOGGING_LEVEL=DEBUG
          export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "VoxCPM E2E Test"
    timeout_in_minutes: 20
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          pip install voxcpm
          export VLLM_LOGGING_LEVEL=DEBUG
          export VLLM_WORKER_MULTIPROC_METHOD=spawn
          pytest -s -v tests/e2e/offline_inference/test_voxcpm.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "VoxCPM2 Native AR E2E Test"
    timeout_in_minutes: 20
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          pip install voxcpm
          export VLLM_LOGGING_LEVEL=DEBUG
          export VLLM_WORKER_MULTIPROC_METHOD=spawn
          pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "OmniVoice E2E Test"
    timeout_in_minutes: 20
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          export VLLM_LOGGING_LEVEL=DEBUG
          pytest -s -v tests/e2e/online_serving/test_omnivoice.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Qwen3-TTS Base E2E Test (ModelRunner V2)"
    depends_on: upload-ready-pipeline
    soft_fail:
      - exit_status: 1
    commands:
      - |
        timeout 20m bash -c '
          export VLLM_LOGGING_LEVEL=DEBUG
          export VLLM_WORKER_MULTIPROC_METHOD=spawn
          export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          export VLLM_OMNI_USE_V2_RUNNER="1"
          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "gpu_1_queue"
    plugins:
      - docker#v5.2.0:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
          shm-size: "8gb"
          environment:
            - "HF_HOME=/fsx/hf_cache"
            - "HF_TOKEN"
          volumes:
            - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Voxtral-TTS E2E Test"
    timeout_in_minutes: 20
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          export VLLM_LOGGING_LEVEL=DEBUG
          pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  # - label: "Diffusion Image Edit Test with H100 (1 GPU)"
  #   depends_on: upload-ready-pipeline
  #   commands:
  #     - |
  #       timeout 20m bash -c '
  #         pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
  #       '
  #   agents:
  #     queue: "mithril-h100-pool"
  #   plugins:
  #     - kubernetes:
  #         podSpec:
  #           containers:
  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
  #               resources:
  #                 limits:
  #                   nvidia.com/gpu: 1
  #               volumeMounts:
  #                 - name: devshm
  #                   mountPath: /dev/shm
  #                 - name: hf-cache
  #                   mountPath: /root/.cache/huggingface
  #               env:
  #                 - name: HF_HOME
  #                   value: /root/.cache/huggingface
  #           nodeSelector:
  #             node.kubernetes.io/instance-type: gpu-h100-sxm
  #           volumes:
  #             - name: devshm
  #               emptyDir:
  #                 medium: Memory
  #             - name: hf-cache
  #               hostPath:
  #                 path: /mnt/hf-cache
  #                 type: DirectoryOrCreate

  - label: "Bagel Text2Img Model Test with H100"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 30m bash -c '
          export VLLM_TEST_CLEAN_GPU_MEMORY=1
          pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: "Bagel Img2Img Model Test with H100"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 30m bash -c '
          export VLLM_TEST_CLEAN_GPU_MEMORY=1
          pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: "Bagel Online Serving Test with H100"
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 40m bash -c '
          export VLLM_TEST_CLEAN_GPU_MEMORY=1
          export VLLM_IMAGE_FETCH_TIMEOUT=60
          pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - label: "CosyVoice3-TTS E2E Test"
    timeout_in_minutes: 20
    depends_on: upload-ready-pipeline
    commands:
      - |
        timeout 20m bash -c '
          pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
        '
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate