467 lines
19 KiB
YAML
467 lines
19 KiB
YAML
steps:
|
|
- label: ":full_moon: Omni Model Test with H100"
|
|
timeout_in_minutes: 90
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "advanced_model" --run-level "advanced_model"
|
|
EXIT1=$$?
|
|
pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
|
|
EXIT2=$$?
|
|
pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
|
|
EXIT3=$$?
|
|
pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
|
|
EXIT4=$$?
|
|
exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4))
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Omni Model Test with L4"
|
|
timeout_in_minutes: 90
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
- pytest -s -v tests/examples/ -m "advanced_model and L4 and omni" --run-level "advanced_model"
|
|
- pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and L4 and omni" --run-level "advanced_model"
|
|
agents:
|
|
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
|
plugins:
|
|
- docker#v5.2.0:
|
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
always-pull: true
|
|
propagate-environment: true
|
|
shm-size: "8gb"
|
|
environment:
|
|
- "HF_HOME=/fsx/hf_cache"
|
|
- "HF_TOKEN"
|
|
volumes:
|
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
|
|
- label: ":full_moon: Diffusion Model Test with H100"
|
|
timeout_in_minutes: 120
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion" -m "advanced_model and diffusion and H100" --run-level "advanced_model"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Diffusion Model Wan22 completed Test with H100"
|
|
timeout_in_minutes: 90
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -m "advanced_model" --run-level "advanced_model"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Diffusion Model Test with L4"
|
|
timeout_in_minutes: 60
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and diffusion and L4" --run-level "advanced_model"
|
|
agents:
|
|
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
|
plugins:
|
|
- docker#v5.2.0:
|
|
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
always-pull: true
|
|
propagate-environment: true
|
|
shm-size: "8gb"
|
|
environment:
|
|
- "HF_HOME=/fsx/hf_cache"
|
|
- "HF_TOKEN"
|
|
volumes:
|
|
- "/fsx/hf_cache:/fsx/hf_cache"
|
|
|
|
|
|
- label: ":full_moon: Documentation Example Code Test with H100"
|
|
timeout_in_minutes: 60
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
|
|
- pytest -s -v tests/examples/online_serving/test_text_to_image.py tests/examples/offline_inference/test_text_to_image.py -m "advanced_model and example and H100" --run-level "advanced_model"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Omni Model Perf Test & Test Case Statistics"
|
|
key: nightly-omni-performance
|
|
timeout_in_minutes: 180
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
- python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: GEBench Accuracy Test with H100"
|
|
key: nightly-gebench-accuracy
|
|
timeout_in_minutes: 60
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level advanced_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
|
|
- buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 1
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: GEdit-Bench Accuracy Test with H100"
|
|
key: nightly-gedit-bench-accuracy
|
|
timeout_in_minutes: 60
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level advanced_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
|
|
- buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
|
|
- buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 1
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
|
|
value: "120"
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Wan22 I2V Accuracy Test with H100"
|
|
key: nightly-wan22-i2v-accuracy
|
|
timeout_in_minutes: 180
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py --run-level advanced_model
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:c392ce21e9cf9ea65c52b866447793db10e0261c
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
- label: ":full_moon: Qwen-Image Diffusion Perf Test with H100"
|
|
key: nightly-qwen-image-performance
|
|
timeout_in_minutes: 300
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
commands:
|
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
- export CACHE_DIT_VERSION=1.3.0
|
|
- pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/benchmark_results_*.json"
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
agents:
|
|
queue: "mithril-h100-pool"
|
|
plugins:
|
|
- kubernetes:
|
|
podSpec:
|
|
containers:
|
|
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 4
|
|
volumeMounts:
|
|
- name: devshm
|
|
mountPath: /dev/shm
|
|
- name: hf-cache
|
|
mountPath: /root/.cache/huggingface
|
|
env:
|
|
- name: HF_HOME
|
|
value: /root/.cache/huggingface
|
|
- name: HF_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: hf-token-secret
|
|
key: token
|
|
nodeSelector:
|
|
node.kubernetes.io/instance-type: gpu-h100-sxm
|
|
volumes:
|
|
- name: devshm
|
|
emptyDir:
|
|
medium: Memory
|
|
- name: hf-cache
|
|
hostPath:
|
|
path: /mnt/hf-cache
|
|
type: DirectoryOrCreate
|
|
|
|
# No need to run this step for PRs with label nightly-test
|
|
- label: ":email: Nightly Collection & Email"
|
|
key: nightly-perf-distribution
|
|
depends_on:
|
|
- nightly-omni-performance
|
|
- nightly-qwen-image-performance
|
|
if: build.env("NIGHTLY") == "1"
|
|
commands:
|
|
- pip install openpyxl
|
|
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
|
|
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
|
|
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance
|
|
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-qwen-image-performance
|
|
- buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-omni-performance
|
|
- python tools/nightly/generate_nightly_perf_excel.py
|
|
- python tools/nightly/generate_nightly_perf_html.py
|
|
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
|
|
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
|
agents:
|
|
queue: "cpu_queue_premerge"
|