582 lines
28 KiB
YAML
582 lines
28 KiB
YAML
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
|
HF_HUB_ETAG_TIMEOUT: 60
|
|
|
|
steps:
|
|
# Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
|
|
- group: ":card_index_dividers: Omni Model Test"
|
|
key: nightly-omni-test-group
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
|
steps:
|
|
|
|
- label: ":full_moon: Omni · Function Test with H100"
|
|
timeout_in_minutes: 120
|
|
commands:
|
|
- pytest -sv tests/e2e/online_serving/test_minicpmo_4_5_expansion.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/offline_inference/test_minicpmo_4_5.py --run-level "full_model" -m "full_model and H100 and omni"
|
|
mirror_hardwares: h100_2
|
|
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
|
|
timeout_in_minutes: 50
|
|
commands:
|
|
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
|
|
mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Omni · Doc Test with L4"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
# - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
|
|
# mirror_hardwares: l4_4
|
|
|
|
# - label: ":full_moon: Omni · Doc Test with H100"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Omni · Accuracy Test"
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export SEED_TTS_WER_EVAL=1
|
|
# - export SEED_TTS_SIM_EVAL=1
|
|
# - export SEED_TTS_EVAL_DEVICE=cuda:1
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_2
|
|
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
|
timeout_in_minutes: 180
|
|
commands:
|
|
- export SEED_TTS_WER_EVAL=1
|
|
- export SEED_TTS_SIM_EVAL=1
|
|
- export SEED_TTS_EVAL_DEVICE=cuda:1
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
|
exit $$EXIT
|
|
mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
|
# key: nightly-omni-performance-no-async-chunk
|
|
# timeout_in_minutes: 300
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Omni · Perf Test · Async Chunk"
|
|
# key: nightly-omni-performance-async-chunk
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Omni · Perf Test · vLLM Text"
|
|
# key: nightly-omni-performance-vllm-text
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Omni · Perf Test · Multi-Replica"
|
|
# key: nightly-omni-performance-multi-replicas
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_3
|
|
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
|
key: nightly-omni-performance-minicpmo-4-5
|
|
timeout_in_minutes: 180
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "cuda and H100" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
mirror_hardwares: h100_1
|
|
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
|
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
|
|
timeout_in_minutes: 180
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
|
|
# timeout_in_minutes: 45
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
|
|
# mirror_hardwares: h100_4
|
|
|
|
# - group: ":card_index_dividers: TTS Model Test"
|
|
# key: nightly-tts-test-group
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
|
# steps:
|
|
|
|
# - label: ":full_moon: TTS · Function Test with L4"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
# - export VLLM_USE_DEEP_GEMM="0"
|
|
# - export VLLM_MOE_USE_DEEP_GEMM="0"
|
|
# - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
|
# mirror_hardwares: l4_1
|
|
|
|
# - label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
|
# timeout_in_minutes: 30
|
|
# commands:
|
|
# - |
|
|
# timeout 30m bash -c "
|
|
# export VLLM_LOGGING_LEVEL=DEBUG
|
|
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
|
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
|
# pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
|
# "
|
|
# mirror_hardwares: l4_1
|
|
|
|
# - label: ":full_moon: TTS · Perf Test"
|
|
# key: nightly-tts-performance
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
|
# key: nightly-tts-performance-voxcpm2
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
|
|
# key: nightly-tts-performance-higgs-audio-v3
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export VLLM_USE_DEEP_GEMM="0"
|
|
# - export VLLM_MOE_USE_DEEP_GEMM="0"
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
|
|
# key: nightly-tiny-model-test-group
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: >-
|
|
# build.env("NIGHTLY") == "1" ||
|
|
# build.pull_request.labels includes "nightly-test"
|
|
# steps:
|
|
|
|
# - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
|
|
# timeout_in_minutes: 30
|
|
# # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
|
|
# # so nightly only adds the multi-GPU parallelism configurations.
|
|
# #
|
|
# # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
|
|
# # --run-level `core_model` is used to indicate that we should use tiny model weights
|
|
# # for the CI, since these tests should pass irrespective of whether or not we use
|
|
# # the full weights or not.
|
|
# commands:
|
|
# - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
|
|
# mirror_hardwares: l4_4
|
|
|
|
# - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
|
# key: nightly-diffusion-x2iat-group
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: >-
|
|
# build.env("NIGHTLY") == "1" ||
|
|
# build.pull_request.labels includes "nightly-test" ||
|
|
# build.pull_request.labels includes "diffusion-x2iat-test"
|
|
# steps:
|
|
# # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
|
|
# # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv
|
|
# tests/e2e/online_serving/test_qwen_image_expansion.py
|
|
# tests/e2e/online_serving/test_qwen_image_edit_expansion.py
|
|
# tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
|
# tests/e2e/online_serving/test_boogu_image.py
|
|
# tests/e2e/online_serving/test_boogu_image_edit.py
|
|
# tests/e2e/online_serving/test_boogu_image_expansion.py
|
|
# tests/e2e/online_serving/test_boogu_image_edit_expansion.py
|
|
# tests/e2e/online_serving/test_bagel_expansion.py
|
|
# tests/e2e/online_serving/test_krea2_expansion.py
|
|
# tests/e2e/offline_inference/test_krea2_expansion.py
|
|
# -m "full_model and diffusion and H100 and not distributed_cuda"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv
|
|
# tests/e2e/online_serving/test_qwen_image_expansion.py
|
|
# tests/e2e/online_serving/test_qwen_image_edit_expansion.py
|
|
# tests/e2e/offline_inference/test_krea2_expansion.py
|
|
# -m "full_model and diffusion and H100 and distributed_cuda"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv
|
|
# tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
|
# -m "full_model and diffusion and H100 and distributed_cuda"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv
|
|
# tests/e2e/online_serving/test_bagel_expansion.py
|
|
# -m "full_model and diffusion and H100 and distributed_cuda"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv
|
|
# tests/e2e/online_serving/test_glm_image_expansion.py
|
|
# tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
|
|
# -m "full_model and diffusion and H100"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
|
# mirror_hardwares: l4_4
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
|
|
# timeout_in_minutes: 60
|
|
# commands:
|
|
# - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
|
|
# - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
|
|
# - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
|
|
# - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
|
|
# - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
|
|
# mirror_hardwares: h100_4
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
|
# key: nightly-diffusion-x2iat-performance-qwen-image
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - export CACHE_DIT_VERSION=1.3.0
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_4
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
|
|
# key: nightly-diffusion-x2iat-performance-qwen-image-edit
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - export CACHE_DIT_VERSION=1.3.0
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_4
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
|
|
# key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - export CACHE_DIT_VERSION=1.3.0
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_4
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
|
|
# key: nightly-diffusion-x2iat-performance-qwen-image-layered
|
|
# timeout_in_minutes: 120
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - export CACHE_DIT_VERSION=1.3.0
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
|
|
# key: nightly-diffusion-x2iat-performance-bagel
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - export CACHE_DIT_VERSION=1.3.0
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
|
|
# EXIT=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# exit $$EXIT
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - group: ":card_index_dividers: Diffusion X2V Model Test"
|
|
# key: nightly-diffusion-x2v-group
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: >-
|
|
# build.env("NIGHTLY") == "1" ||
|
|
# build.pull_request.labels includes "nightly-test" ||
|
|
# build.pull_request.labels includes "diffusion-x2v-test"
|
|
# steps:
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model"
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Doc Test"
|
|
# timeout_in_minutes: 60
|
|
# commands:
|
|
# - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Other Function Test"
|
|
# timeout_in_minutes: 90
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
|
# - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
|
|
# - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
|
|
# - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
|
# - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
|
|
# - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":full_moon: Diffusion X2V · Perf Test"
|
|
# key: nightly-diffusion-x2v-performance
|
|
# timeout_in_minutes: 180
|
|
# commands:
|
|
# - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
# - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
|
# - |
|
|
# set +e
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
|
|
# EXIT1=$$?
|
|
# pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
|
|
# EXIT2=$$?
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
# buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
# if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
|
|
# exit $$EXIT2
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - group: ":card_index_dividers: Diffusion Test"
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
|
|
# steps:
|
|
|
|
# - label: ":full_moon: Diffusion · Offloader Test"
|
|
# timeout_in_minutes: 40
|
|
# commands:
|
|
# - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda"
|
|
# mirror_hardwares: l4_1
|
|
|
|
# - label: ":full_moon: Diffusion · Quantization Test with H100"
|
|
# commands:
|
|
# - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
|
|
# mirror_hardwares: h100_1
|
|
|
|
# - label: ":full_moon: Diffusion · Quantization Test with L4"
|
|
# timeout_in_minutes: 60
|
|
# commands:
|
|
# - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
|
|
# mirror_hardwares: l4_1
|
|
|
|
# - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test"
|
|
# timeout_in_minutes: 60
|
|
# commands:
|
|
# - >-
|
|
# pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py
|
|
# -m "full_model and diffusion and H100 and distributed_cuda"
|
|
# --run-level "full_model"
|
|
# mirror_hardwares: h100_2
|
|
|
|
# - label: ":bar_chart: Testcase Statistics"
|
|
# key: nightly-testcase-statistics
|
|
# timeout_in_minutes: 120
|
|
# depends_on: upload-nightly-pipeline
|
|
# if: build.env("NIGHTLY") == "1"
|
|
# commands:
|
|
# - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
|
|
# - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
|
# mirror_hardwares: h100_1
|
|
|
|
# # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
|
|
# # Do not add new depends_on entries or extend this step.
|
|
# - label: ":email: Nightly Collection & Email"
|
|
# key: nightly-perf-distribution
|
|
# depends_on:
|
|
# - nightly-omni-performance-no-async-chunk
|
|
# - nightly-omni-performance-async-chunk
|
|
# - nightly-omni-performance-vllm-text
|
|
# - nightly-omni-performance-multi-replicas
|
|
# - nightly-tts-performance
|
|
# - nightly-tts-performance-higgs-audio-v3
|
|
# - nightly-diffusion-x2iat-performance-qwen-image
|
|
# - nightly-diffusion-x2iat-performance-qwen-image-edit
|
|
# - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
|
# - nightly-diffusion-x2iat-performance-qwen-image-layered
|
|
# - nightly-diffusion-x2iat-performance-bagel
|
|
# - nightly-diffusion-x2v-performance
|
|
# - nightly-testcase-statistics
|
|
# if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
|
|
# commands:
|
|
# - pip install openpyxl
|
|
# - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
|
|
# - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
|
|
# - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
|
|
# - python tools/nightly/generate_nightly_perf_excel.py
|
|
# - python tools/nightly/generate_nightly_perf_html.py
|
|
# - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
|
|
# - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
|
|
# - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
|
# agents:
|
|
# queue: "cpu_queue_premerge"
|