296 lines
13 KiB
YAML
296 lines
13 KiB
YAML
steps:
|
|
- group: ":card_index_dividers: Omni Model Test"
|
|
key: nightly-omni-test-group
|
|
depends_on: upload-nightly-pipeline
|
|
if: >-
|
|
build.env("NIGHTLY") == "1" ||
|
|
build.pull_request.labels includes "nightly-test"
|
|
steps:
|
|
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
|
source_file_dependencies: omni_qwen3_omni_perf
|
|
key: nightly-omni-performance-no-async-chunk
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Omni · Perf Test · Async Chunk"
|
|
source_file_dependencies: omni_qwen3_omni_perf
|
|
key: nightly-omni-performance-async-chunk
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
|
source_file_dependencies: omni_minicpmo_4_5_function
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
|
source_file_dependencies: omni_minicpmo_4_5_duplex_function
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- >-
|
|
pytest -s -v
|
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
|
-m "npu and A3 and omni"
|
|
--run-level full_model
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
|
source_file_dependencies: omni_minicpmo_4_5_accuracy
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
# Pin the Seed-TTS scorer to the second card: _get_eval_device() otherwise
|
|
# auto-picks npu:0, which the omni server under test already fills with its
|
|
# KV cache, so Whisper OOMs on `.to(device)` and every clip reports
|
|
# "ASR / WER failed".
|
|
- export SEED_TTS_WER_EVAL=1
|
|
- export SEED_TTS_EVAL_DEVICE=npu:1
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
|
source_file_dependencies: omni_minicpmo_4_5_perf
|
|
key: nightly-omni-performance-minicpmo-4-5-npu
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
|
|
source_file_dependencies: omni_minicpmo_4_5_perf
|
|
key: nightly-omni-performance-minicpmo-4-5-npu-a2
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a2b3_npu_1
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
HF_HUB_OFFLINE: "1"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
|
source_file_dependencies: omni_minicpmo_4_5_duplex_perf
|
|
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- group: ":card_index_dividers: TTS Model Test"
|
|
key: nightly-tts-test-group
|
|
depends_on: upload-nightly-pipeline
|
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
|
steps:
|
|
- label: ":full_moon: TTS · Perf Test"
|
|
source_file_dependencies: tts_qwen3_tts_perf
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|
|
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
|
key: nightly-diffusion-x2iat-group
|
|
depends_on: upload-nightly-pipeline
|
|
if: >-
|
|
build.env("NIGHTLY") == "1" ||
|
|
build.pull_request.labels includes "nightly-test"
|
|
steps:
|
|
- label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
|
|
source_file_dependencies: diffusion_hunyuan_image3_accuracy
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
|
|
HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
|
|
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
|
|
source_file_dependencies: diffusion_hunyuan_image3_perf
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
|
|
source_file_dependencies: diffusion_hunyuan_image3_perf
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
|
|
source_file_dependencies: diffusion_hunyuan_image3_perf
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
exit $$EXIT
|
|
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
|
key: nightly-diffusion-x2v-group
|
|
depends_on: upload-nightly-pipeline
|
|
if: >-
|
|
build.env("NIGHTLY") == "1" ||
|
|
build.pull_request.labels includes "nightly-test"
|
|
steps:
|
|
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
|
|
source_file_dependencies: diffusion_hunyuan_video_function
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a2b3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
HF_HUB_OFFLINE: "1"
|
|
commands:
|
|
- pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model"
|
|
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test"
|
|
source_file_dependencies: diffusion_hunyuan_video_perf
|
|
key: nightly-diffusion-x2v-performance-hunyuanvideo15
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_2
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
HF_HUB_OFFLINE: "1"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
|
exit $$EXIT
|
|
- label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test"
|
|
source_file_dependencies: diffusion_wan22_function
|
|
mirror_hardwares: a2b3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
commands:
|
|
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
|
- label: ":full_moon: Diffusion X2V · Wan 2.2 Perf Test"
|
|
source_file_dependencies: diffusion_wan22_perf
|
|
key: nightly-diffusion-x2v-performance
|
|
timeout_in_minutes: 180
|
|
mirror_hardwares: a3_npu_4
|
|
env:
|
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
|
HF_TOKEN: "${HF_TOKEN}"
|
|
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
|
commands:
|
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
|
- |
|
|
set +e
|
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
|
|
EXIT=$$?
|
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
|
exit $$EXIT
|