vllm-omni/.buildkite/test-merge.yml

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  VLLM_USE_FLASHINFER_MOE_FP16: 0
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Simple · Diffusion & Model Executor Test"
        commands:
          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "Simple · Other Test"
        commands:
          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

  - group: ":card_index_dividers: Diffusion Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Diffusion · LoRA Test"
        timeout_in_minutes: 30
        commands:
          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
        agents:
          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

  - label: "Distributed Test"
    timeout_in_minutes: 60
    commands:
      - |
        timeout 55m bash -c "
          set -e
          export VLLM_IMAGE_FETCH_TIMEOUT=60
          pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
        "
    agents:
      queue: "mithril-h100-pool"
    plugins:
      - kubernetes:
          podSpec:
            containers:
              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                resources:
                  limits:
                    nvidia.com/gpu: 1
                volumeMounts:
                  - name: devshm
                    mountPath: /dev/shm
                  - name: hf-cache
                    mountPath: /root/.cache/huggingface
                env:
                  - name: HF_HOME
                    value: /root/.cache/huggingface
                  - name: HF_TOKEN
                    valueFrom:
                      secretKeyRef:
                        name: hf-token-secret
                        key: token
            nodeSelector:
              node.kubernetes.io/instance-type: gpu-h100-sxm
            volumes:
              - name: devshm
                emptyDir:
                  medium: Memory
              - name: hf-cache
                hostPath:
                  path: /mnt/hf-cache
                  type: DirectoryOrCreate

  - group: ":card_index_dividers: Entrypoints Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Entrypoint Test with H100"
        timeout_in_minutes: 35
        commands:
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Entrypoint Test with L4"
        timeout_in_minutes: 30
        commands:
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
        agents:
          queue: "gpu_4_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

  - group: ":card_index_dividers: E2E Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "TTS · Qwen3-TTS CustomVoice Test"
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "gpu_4_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "TTS · Qwen3-TTS Base Test"
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "gpu_4_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "TTS · Voxtral-TTS Test"
        timeout_in_minutes: 20
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py tests/e2e/online_serving/test_voxtral_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "TTS · GLM-TTS Test"
        timeout_in_minutes: 20
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Omni · Qwen3-Omni Test"
        timeout_in_minutes: 30
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Omni · Mimo Audio Test"
        timeout_in_minutes: 30
        commands:
          - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 2
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Omni · VoxCPM2 Test"
        timeout_in_minutes: 20
        commands:
          - |
            timeout 20m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_WORKER_MULTIPROC_METHOD=spawn
              pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "Omni · Higgs-Audio V2 Test"
        timeout_in_minutes: 30
        commands:
          - |
            timeout 30m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_WORKER_MULTIPROC_METHOD=spawn
              export VLLM_USE_DEEP_GEMM=0
              export VLLM_MOE_USE_DEEP_GEMM=0
              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "gpu_1_queue"
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              shm-size: "8gb"
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "Diffusion · Qwen Image Edit Test"
        timeout_in_minutes: 20
        commands:
          - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Longcat Image Edit Test"
        timeout_in_minutes: 20
        commands:
          - pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Longcat Image Test"
        timeout_in_minutes: 20
        commands:
          - pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Bagel Test"
        timeout_in_minutes: 20
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate


      - label: "Diffusion · Qwen Image Test"
        commands:
          - |
            timeout 40m bash -c '
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            '
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Qwen Image Layered Test"
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Wan22 Test"
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · HunyuanVideo 1.5 Test"
        timeout_in_minutes: 40
        commands:
          - |
            timeout 40m bash -c "
              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 1
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate

      - label: "Diffusion · Z Image Test"
        timeout_in_minutes: 30
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
        agents:
          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "Diffusion · FLUX.2 Klein Test"
        timeout_in_minutes: 30
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
        agents:
          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
        plugins:
          - docker#v5.2.0:
              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
              always-pull: true
              propagate-environment: true
              environment:
                - "HF_HOME=/fsx/hf_cache"
                - "HF_TOKEN"
              volumes:
                - "/fsx/hf_cache:/fsx/hf_cache"

      - label: "Diffusion · SD3 Test"
        timeout_in_minutes: 45
        commands:
          - |
            timeout 45m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_sd3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        agents:
          queue: "mithril-h100-pool"
        plugins:
          - kubernetes:
              podSpec:
                containers:
                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                    resources:
                      limits:
                        nvidia.com/gpu: 4
                    volumeMounts:
                      - name: devshm
                        mountPath: /dev/shm
                      - name: hf-cache
                        mountPath: /root/.cache/huggingface
                    env:
                      - name: HF_HOME
                        value: /root/.cache/huggingface
                      - name: HF_TOKEN
                        valueFrom:
                          secretKeyRef:
                            name: hf-token-secret
                            key: token
                nodeSelector:
                  node.kubernetes.io/instance-type: gpu-h100-sxm
                volumes:
                  - name: devshm
                    emptyDir:
                      medium: Memory
                  - name: hf-cache
                    hostPath:
                      path: /mnt/hf-cache
                      type: DirectoryOrCreate