env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_USE_FLASHINFER_MOE_FP16: 0 HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_ETAG_TIMEOUT: 60 steps: # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel. - group: ":card_index_dividers: Omni Model Test" key: nightly-omni-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - label: ":full_moon: Omni · Function Test with H100" timeout_in_minutes: 90 commands: - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Function Test with L4" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - label: ":full_moon: Omni · Doc Test with L4" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - label: ":full_moon: Omni · Doc Test with H100" timeout_in_minutes: 90 commands: - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Accuracy Test" timeout_in_minutes: 180 commands: - export SEED_TTS_WER_EVAL=1 - export SEED_TTS_EVAL_DEVICE=cuda:1 - | set +e pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model EXIT=$$? buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Perf Test · No Async Chunk" key: nightly-omni-performance-no-async-chunk timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Perf Test · Async Chunk" key: nightly-omni-performance-async-chunk timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Perf Test · vLLM Text" key: nightly-omni-performance-vllm-text timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Perf Test · Multi-Replica" key: nightly-omni-performance-multi-replicas timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 3 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" timeout_in_minutes: 45 commands: - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 4 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Function Test" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - label: ":full_moon: TTS · Perf Test" key: nightly-tts-performance timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: VoxCPM2 · Perf Test" key: nightly-voxcpm2-performance timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below. - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" key: nightly-diffusion-x2iat-group depends_on: upload-nightly-pipeline if: >- build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: # Use explicit file shards so single-card cases do not reserve 2+ GPUs, # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py tests/e2e/online_serving/test_qwen_image_edit_expansion.py tests/e2e/online_serving/test_qwen_image_layered_expansion.py tests/e2e/online_serving/test_longcat_image_expansion.py tests/e2e/online_serving/test_longcat_image_edit_expansion.py tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/online_serving/test_bagel_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py tests/e2e/online_serving/test_qwen_image_edit_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_layered_expansion.py tests/e2e/online_serving/test_longcat_image_expansion.py tests/e2e/online_serving/test_longcat_image_edit_expansion.py tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_bagel_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 60 commands: - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" timeout_in_minutes: 60 commands: - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE value: "120" - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image" key: nightly-diffusion-x2iat-performance-qwen-image timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 4 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" key: nightly-diffusion-x2iat-performance-qwen-image-edit timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 4 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 4 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" key: nightly-diffusion-x2iat-performance-qwen-image-layered timeout_in_minutes: 120 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" key: nightly-diffusion-x2iat-performance-bagel timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test" key: nightly-hunyuan-image3-performance soft_fail: true timeout_in_minutes: 120 if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1" commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - export CACHE_DIT_VERSION=1.3.0 - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json EXIT1=$$? pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json EXIT2=$$? pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json EXIT3=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$((EXIT1 | EXIT2 | EXIT3)) agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 4 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate # Diffusion x2v only (Wan, HunyuanVideo, …). x2i/x2a/x2t live in the X2I group above, not here. - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group depends_on: upload-nightly-pipeline if: >- build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2v-test" steps: - label: ":full_moon: Diffusion X2V · Function Test" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2V · Accuracy Test" timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - | set +e pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json EXIT1=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT1 agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 2 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - group: ":card_index_dividers: Diffusion · Quantization Test" depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: - label: ":full_moon: Quantization Test with H100" commands: - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate - label: ":full_moon: Quantization Test with L4" timeout_in_minutes: 60 commands: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" agents: queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: - docker#v5.2.0: image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT always-pull: true propagate-environment: true environment: - "HF_HOME=/fsx/hf_cache" - "HF_TOKEN" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - label: ":bar_chart: Testcase Statistics" key: nightly-testcase-statistics timeout_in_minutes: 120 depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" commands: - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" agents: queue: "mithril-h100-pool" plugins: - kubernetes: podSpec: containers: - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT resources: limits: nvidia.com/gpu: 1 volumeMounts: - name: devshm mountPath: /dev/shm - name: hf-cache mountPath: /root/.cache/huggingface env: - name: HF_HOME value: /root/.cache/huggingface - name: HF_TOKEN valueFrom: secretKeyRef: name: hf-token-secret key: token nodeSelector: node.kubernetes.io/instance-type: gpu-h100-sxm volumes: - name: devshm emptyDir: medium: Memory - name: hf-cache hostPath: path: /mnt/hf-cache type: DirectoryOrCreate # No need to run this step for PRs with label nightly-test - label: ":email: Nightly Collection & Email" key: nightly-perf-distribution depends_on: - nightly-omni-performance-no-async-chunk - nightly-omni-performance-async-chunk - nightly-omni-performance-vllm-text - nightly-omni-performance-multi-replicas - nightly-tts-performance - nightly-diffusion-x2iat-performance-qwen-image - nightly-diffusion-x2iat-performance-qwen-image-edit - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - nightly-diffusion-x2iat-performance-qwen-image-layered - nightly-diffusion-x2iat-performance-bagel - nightly-diffusion-x2v-performance - nightly-testcase-statistics if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" commands: - pip install openpyxl - export DEFAULT_INPUT_DIR=tests/dfx/perf/results - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - python tools/nightly/generate_nightly_perf_excel.py - python tools/nightly/generate_nightly_perf_html.py - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx" - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" agents: queue: "cpu_queue_premerge"