# CUDA CI L5 (`slow` / reliability / extra coverage). # Uploaded by Upload Weekly Pipeline: WEEKLY=1 / NON_CRITICAL=1, or PR label `weekly-test`. env: VLLM_WORKER_MULTIPROC_METHOD: spawn HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_ETAG_TIMEOUT: 60 steps: - label: "Simple · CPU Coverage Test" key: weekly-simple-cpu-coverage depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" timeout_in_minutes: 60 commands: - | set +e REPORT="coverage-core-model-cpu-$${BUILDKITE_STEP_ID:-local}.xml" pytest -sv tests/ -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:$${REPORT} EXIT=$$? gzip -9 -f "$${REPORT}" buildkite-agent artifact upload "$${REPORT}.gz" exit $$EXIT mirror_hardwares: l4_1 - group: "Reliability Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" steps: - label: "Reliability Test · qwen3-omni" source_file_dependencies: omni_qwen3_omni_reliability timeout_in_minutes: 180 commands: - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" mirror_hardwares: h100_2 - label: "Reliability Test · wan22" source_file_dependencies: diffusion_wan22_reliability timeout_in_minutes: 180 commands: - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" mirror_hardwares: h100_2 - label: "Reliability Test · hunyuan-image" source_file_dependencies: diffusion_hunyuan_image3_reliability timeout_in_minutes: 180 commands: - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" mirror_hardwares: h100_4 - label: "Reliability Test · Invalid parameters · H100 · Single-GPU" source_file_dependencies: reliability_invalid_param_h100 timeout_in_minutes: 90 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1" mirror_hardwares: h100_1 - label: "Reliability Test · Invalid parameters · H100 · 2-GPU" source_file_dependencies: reliability_invalid_param_h100 timeout_in_minutes: 90 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2" mirror_hardwares: h100_2 - label: "Reliability Test · Invalid parameters · L4" source_file_dependencies: reliability_invalid_param_l4 timeout_in_minutes: 120 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4" mirror_hardwares: l4_1 - group: "Perf Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" steps: - label: ":full_moon: Omni · Perf Test · vLLM Text" source_file_dependencies: omni_qwen3_omni_perf key: weekly-omni-performance-vllm-text timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Async Chunk · Random" key: weekly-omni-performance-async-chunk-random timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT mirror_hardwares: h100_2 - label: ":full_moon: Omni · Perf Test · Multi-Replica" source_file_dependencies: omni_qwen3_omni_perf key: weekly-omni-performance-multi-replicas timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT mirror_hardwares: h100_3 - group: ":card_index_dividers: E2E Tests" depends_on: upload-weekly-pipeline if: build.env("NON_CRITICAL") == "1" steps: # Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card. - label: "Omni · L4 · 2-GPU" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model" mirror_hardwares: l4_2 - label: "Omni · L4 · 4-GPU" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model" - >- pytest -s -v tests/e2e/ --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "full_model" mirror_hardwares: l4_4 - label: "Omni · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/ -m "slow and H100 and omni and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Omni · H100 · 4-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/ -m "slow and H100 and omni and cards_4" --run-level "full_model" mirror_hardwares: h100_4 - label: "TTS · L4" timeout_in_minutes: 120 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - | failed=0 for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 done exit $$failed mirror_hardwares: l4_1 - label: "TTS · H100" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - >- pytest -sv tests/e2e/ -m "slow and H100 and tts" --run-level "full_model" mirror_hardwares: h100_1 # Own step: the image has no NanoCodec, so this file import-skips out of # the mark-selected TTS steps above. The overrides keep nemo_toolkit[tts] # from downgrading transformers; see # examples/offline_inference/text_to_speech/README.md. - label: "TTS · Gepard-1.0 · L4" timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt - >- pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py tests/e2e/online_serving/test_gepard_tts.py -m "tts and cuda" --run-level "full_model" mirror_hardwares: l4_1 - label: "Diffusion · L4 · Single-GPU" timeout_in_minutes: 60 commands: - >- pytest -sv tests/e2e/ -m "slow and diffusion and L4 and cards_1" --run-level "full_model" mirror_hardwares: l4_1 - label: "Diffusion · L4 · 2-GPU" timeout_in_minutes: 60 commands: - >- pytest -sv tests/e2e/ -m "slow and diffusion and L4 and cards_2" --run-level "full_model" mirror_hardwares: l4_2 - label: "Diffusion · L4 · 4-GPU" timeout_in_minutes: 60 commands: - >- pytest -sv tests/e2e/ -m "slow and diffusion and L4 and cards_4" --run-level "full_model" mirror_hardwares: l4_4 # Split the old shared H100 Single-GPU sweep: one pytest process collected # every slow+cards_1 diffusion file and exceeded 150 minutes. - label: "Diffusion · Qwen-Image · H100 · Single-GPU" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_edit.py tests/e2e/online_serving/test_qwen_image_edit_expansion.py tests/e2e/online_serving/test_qwen_image_layered.py tests/e2e/online_serving/test_qwen_image_layered_expansion.py tests/e2e/accuracy/test_qwen_image_edit.py tests/e2e/accuracy/test_qwen_image_layered.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Wan2.1 VACE · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_wan_2_1_vace_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · LongCat · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_longcat_image_expansion.py tests/e2e/online_serving/test_longcat_image_edit_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Bagel · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_bagel.py tests/e2e/online_serving/test_bagel_expansion.py tests/e2e/offline_inference/test_bagel.py tests/e2e/offline_inference/test_bagel_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Boogu · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_boogu_image.py tests/e2e/online_serving/test_boogu_image_edit.py tests/e2e/online_serving/test_boogu_image_expansion.py tests/e2e/online_serving/test_boogu_image_edit_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Flux · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/offline_inference/test_flux1_schnell_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · LTX · H100 · Single-GPU" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_ltx.py tests/e2e/online_serving/test_ltx25.py tests/e2e/accuracy/ltx/test_ltx_official_similarity.py tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Krea2 · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_krea2_expansion.py tests/e2e/offline_inference/test_krea2_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · HiDream-I1 · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_hidream_i1_full_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · SANA-Video · H100 · Single-GPU" source_file_dependencies: diffusion_sana_video_function timeout_in_minutes: 180 commands: - export SANA_VIDEO_GOLDEN_BASE_URL=https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/sana-video/v2 - >- pytest -sv tests/e2e/online_serving/test_sana_video_expansion.py tests/e2e/accuracy/sana_video/ -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · SenseNova-U1 · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/offline_inference/test_sensenova_u1_expansion.py tests/e2e/offline_inference/test_sensenova_u1_text2img_expansion.py tests/e2e/offline_inference/test_sensenova_u1_img2img_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · MammothModa2 · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/offline_inference/test_mammoth_moda2_expansion.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 - label: "Diffusion · Helios · H100 · Single-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/accuracy/test_video_streaming_output_similarity.py -m "slow and diffusion and H100 and cards_1" --run-level "full_model" mirror_hardwares: h100_1 # Split the old shared H100 2-GPU sweep: one pytest process collected every # slow+cards_2 diffusion file and exceeded 90 minutes. - label: "Diffusion · Qwen-Image · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_edit_expansion.py tests/e2e/online_serving/test_qwen_image_layered_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · Wan2.1 VACE · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_wan_2_1_vace_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · LongCat · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_longcat_image_expansion.py tests/e2e/online_serving/test_longcat_image_edit_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · Bagel · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_bagel_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · Boogu · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_boogu_image_expansion.py tests/e2e/online_serving/test_boogu_image_edit_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · Flux · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/offline_inference/test_flux1_kontext_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · GLM-Image · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · LTX · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_ltx.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · Krea2 · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/offline_inference/test_krea2_expansion.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · DreamZero · H100 · 2-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/online_serving/test_dreamzero_expansion.py tests/e2e/accuracy/test_dreamzero.py -m "slow and diffusion and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 - label: "Diffusion · H100 · 3-GPU" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/ -m "slow and diffusion and H100 and cards_3" --run-level "full_model" mirror_hardwares: h100_3 - label: "Diffusion · H100 · 4-GPU" timeout_in_minutes: 90 commands: - >- pytest -sv tests/e2e/ -m "slow and diffusion and H100 and cards_4" --run-level "full_model" --ignore=tests/e2e/online_serving/test_magi2.py mirror_hardwares: h100_4 # Dedicated: HiDream-O1 accuracy needs a long TP2 run; the shared H100 # 2-GPU sweep is only 90 minutes. - label: "Diffusion · HiDream-O1-Image Accuracy · H100 · 2-GPU" timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m "slow and diffusion and cuda and H100 and cards_2" --run-level "full_model" mirror_hardwares: h100_2 # Dedicated: Boogu CFG-parallel size 3 needs 3 GPUs; weekly has no H100 3-GPU sweep. - label: "Diffusion · Boogu Edit CFG · H100 · 3-GPU" timeout_in_minutes: 120 commands: - >- pytest -sv tests/e2e/online_serving/test_boogu_image_edit_expansion.py -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full" -m "slow and diffusion and cuda and H100 and cards_3" --run-level "full_model" mirror_hardwares: h100_3 # Dedicated: MAGI-2 Preview serving smoke needs ~30 min server init; the # shared H100 4-GPU sweep is only 90 minutes. - label: "Diffusion · MAGI-2 Preview Serving Smoke · H100 · 4-GPU" timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "slow and diffusion and cuda and H100 and cards_4" --run-level "full_model" mirror_hardwares: h100_4