diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index ed112b84b..0c5d94d81 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -159,19 +159,7 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py - mirror_hardwares: l4_1 - - - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' - " + - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy mirror_hardwares: l4_1 - label: ":full_moon: TTS · Perf Test" @@ -188,35 +176,6 @@ steps: exit $$EXIT mirror_hardwares: h100_2 - - label: ":full_moon: TTS · VoxCPM2 · Perf Test" - key: nightly-tts-performance-voxcpm2 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 - - - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" - key: nightly-tts-performance-higgs-audio-v3 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_USE_DEEP_GEMM="0" - - export VLLM_MOE_USE_DEEP_GEMM="0" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 - - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" key: nightly-tiny-model-test-group depends_on: upload-nightly-pipeline @@ -246,8 +205,7 @@ steps: build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" steps: - # Use explicit file shards so single-card cases do not reserve 2+ GPUs, - # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. + # Use explicit file shards so single-card cases do not reserve 2+ GPUs. - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" timeout_in_minutes: 120 @@ -255,15 +213,6 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - tests/e2e/online_serving/test_boogu_image.py - tests/e2e/online_serving/test_boogu_image_edit.py - tests/e2e/online_serving/test_boogu_image_expansion.py - tests/e2e/online_serving/test_boogu_image_edit_expansion.py - tests/e2e/online_serving/test_bagel_expansion.py - tests/e2e/online_serving/test_krea2_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" mirror_hardwares: h100_1 @@ -274,47 +223,14 @@ steps: - >- pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py -m "full_model and diffusion and H100 and distributed_cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" - timeout_in_minutes: 90 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 - - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_bagel_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 - - - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_glm_image_expansion.py - tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py - -m "full_model and diffusion and H100" - --run-level "full_model" - mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 120 commands: - - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy + - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy mirror_hardwares: l4_4 - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" @@ -327,7 +243,7 @@ steps: timeout_in_minutes: 180 commands: - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 - - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model mirror_hardwares: h100_1 @@ -355,70 +271,6 @@ steps: exit $$EXIT mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" - key: nightly-diffusion-x2iat-performance-qwen-image-edit - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" - key: nightly-diffusion-x2iat-performance-qwen-image-layered - timeout_in_minutes: 120 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 - - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" - key: nightly-diffusion-x2iat-performance-bagel - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 - - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group depends_on: upload-nightly-pipeline @@ -440,12 +292,6 @@ steps: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Doc Test" timeout_in_minutes: 60 commands: @@ -456,15 +302,8 @@ steps: timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LTX-2.5 Online Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/e2e/online_serving/test_ltx25.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" timeout_in_minutes: 180 commands: @@ -483,23 +322,8 @@ steps: commands: - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LTX Accuracy Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model - - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model - mirror_hardwares: h100_1 - - - label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 @@ -575,16 +399,6 @@ steps: - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" mirror_hardwares: l4_1 - - label: ":bar_chart: Testcase Statistics" - key: nightly-testcase-statistics - timeout_in_minutes: 120 - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" - commands: - - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" - mirror_hardwares: h100_1 - # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. # Do not add new depends_on entries or extend this step. - label: ":email: Nightly Collection & Email" @@ -595,32 +409,20 @@ steps: - nightly-omni-performance-vllm-text - nightly-omni-performance-multi-replicas - nightly-tts-performance - - nightly-tts-performance-higgs-audio-v3 - nightly-diffusion-x2iat-performance-qwen-image - - nightly-diffusion-x2iat-performance-qwen-image-edit - - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - nightly-diffusion-x2iat-performance-qwen-image-layered - - nightly-diffusion-x2iat-performance-bagel - nightly-diffusion-x2v-performance - - nightly-testcase-statistics if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" commands: - pip install openpyxl - export DEFAULT_INPUT_DIR=tests/dfx/perf/results - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3 - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - python tools/nightly/generate_nightly_perf_excel.py - python tools/nightly/generate_nightly_perf_html.py - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 0fd3a06ce..5ddbf22ce 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -149,4 +149,4 @@ steps: pytest -sv tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda" --run-level "full_model" - mirror_hardwares: h100_2 + mirror_hardwares: h100_4 diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 050cef4f7..2cdf0cd54 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -96,33 +96,6 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" - key: nightly-diffusion-x2iat-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2iat-test" - steps: - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - timeout_in_minutes: 180 - mirror_hardwares: a3_npu_8 - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" - HF_TOKEN: "${HF_TOKEN}" - DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export CACHE_DIT_VERSION=1.5.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - group: ":card_index_dividers: Diffusion I2I Model Test" key: nightly-HunyuanImage3-x2i-test-group depends_on: upload-nightly-pipeline diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 9f6ff0241..9dda3148d 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task", @@ -61,7 +61,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task", @@ -113,7 +113,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task", @@ -164,7 +164,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task", diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 75e5983b0..1197c88f1 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "tts" ], "server_params": { diff --git a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json index 1a0f31e98..9b0634a20 100644 --- a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json +++ b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline with enforce-eager (no torch.compile)", @@ -64,7 +64,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device with torch.compile (default, no enforce-eager)", @@ -117,7 +117,7 @@ "num_cards": 2 } }, - "full_model", + "local_model", "diffusion" ], "description": "CFG-parallel=2 with enforce-eager", @@ -172,7 +172,7 @@ "num_cards": 2 } }, - "full_model", + "local_model", "diffusion" ], "description": "CFG-parallel=2 with torch.compile", @@ -226,7 +226,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "CacheDiT with enforce-eager", diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index dcd2188c5..fd8e437fd 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline (two input images)", @@ -76,7 +76,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", @@ -125,7 +125,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + CacheDiT", diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 686bd6160..62ee70aa2 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline", @@ -73,7 +73,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", @@ -121,7 +121,7 @@ "num_cards": 4 } }, - "full_model", + "local_model", "diffusion" ], "description": "Ulysses SP=2 + CFG=2 + CacheDiT", diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index bc4d93a24..b5fa2815b 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "diffusion" ], "description": "Single-device baseline", diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index e5163e5bb..1756bc206 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -10,7 +10,7 @@ "num_cards": 1 } }, - "full_model", + "local_model", "tts" ], "server_params": { diff --git a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py index 9f1623c8a..d72d14022 100644 --- a/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py @@ -463,7 +463,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f } -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) @@ -591,7 +591,7 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path assert audio_metrics["relative_l2"] <= LTX25_AUDIO_RELATIVE_L2_THRESHOLD -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) diff --git a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py index 8a3be4a99..59a8540aa 100644 --- a/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py +++ b/tests/e2e/accuracy/ltx/test_ltx_official_similarity.py @@ -544,7 +544,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f @pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.benchmark @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index 9ee9c2c87..935a032d8 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -17,7 +17,7 @@ from tests.helpers.mark import hardware_test from tests.helpers.media import get_asset_path from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.diffusion] SINGLE_MODEL = "Qwen/Qwen-Image-Edit" @@ -253,6 +253,7 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image ) +@pytest.mark.slow @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_single_matches_diffusers( @@ -278,6 +279,7 @@ def test_qwen_image_edit_single_matches_diffusers( ) +@pytest.mark.slow @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: diff --git a/tests/e2e/accuracy/test_qwen_image_layered.py b/tests/e2e/accuracy/test_qwen_image_layered.py index f2bc1b6aa..3ad8d3328 100644 --- a/tests/e2e/accuracy/test_qwen_image_layered.py +++ b/tests/e2e/accuracy/test_qwen_image_layered.py @@ -17,7 +17,7 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] MODEL_ID = "Qwen/Qwen-Image-Layered" diff --git a/tests/e2e/accuracy/test_video_streaming_output_similarity.py b/tests/e2e/accuracy/test_video_streaming_output_similarity.py index 799e6e125..1aa7cdfc2 100644 --- a/tests/e2e/accuracy/test_video_streaming_output_similarity.py +++ b/tests/e2e/accuracy/test_video_streaming_output_similarity.py @@ -11,7 +11,7 @@ from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_vi from tests.helpers.mark import hardware_marks from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "BestWishYsh/Helios-Distilled" PROMPT = "A serene lakeside sunrise with mist over the water." diff --git a/tests/e2e/offline_inference/test_bagel_expansion.py b/tests/e2e/offline_inference/test_bagel_expansion.py index e8ac6c19f..4c31be722 100644 --- a/tests/e2e/offline_inference/test_bagel_expansion.py +++ b/tests/e2e/offline_inference/test_bagel_expansion.py @@ -138,7 +138,7 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest: pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py index 28eaab9b3..86b8cee8a 100644 --- a/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py @@ -109,7 +109,7 @@ def _diffusion_sampling_params( pytestmark = [ - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.diffusion, ] diff --git a/tests/e2e/offline_inference/test_moss_tts_realtime.py b/tests/e2e/offline_inference/test_moss_tts_realtime.py index 892a395ba..34d36118d 100644 --- a/tests/e2e/offline_inference/test_moss_tts_realtime.py +++ b/tests/e2e/offline_inference/test_moss_tts_realtime.py @@ -96,7 +96,7 @@ def _get_test_config() -> str: pytestmark = [ pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"), - pytest.mark.full_model, + pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize( "omni_runner", @@ -271,7 +271,6 @@ def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor # --------------------------------------------------------------------------- -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) -> None: """MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio.""" @@ -283,7 +282,6 @@ def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) assert not torch.all(audio == 0), "Audio is silence" -@pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_moss_tts_realtime_chinese(omni_runner: OmniRunner, ref_audio_path: str) -> None: """MossTTSRealtime: Chinese input produces non-empty audio.""" diff --git a/tests/e2e/online_serving/test_bagel_expansion.py b/tests/e2e/online_serving/test_bagel_expansion.py index fa17791e5..3308efdcb 100644 --- a/tests/e2e/online_serving/test_bagel_expansion.py +++ b/tests/e2e/online_serving/test_bagel_expansion.py @@ -22,7 +22,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution." NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark" diff --git a/tests/e2e/online_serving/test_bagel_multi_replicas.py b/tests/e2e/online_serving/test_bagel_multi_replicas.py index cb7aeeb94..6409ddd58 100644 --- a/tests/e2e/online_serving/test_bagel_multi_replicas.py +++ b/tests/e2e/online_serving/test_bagel_multi_replicas.py @@ -1,6 +1,8 @@ -"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving. +"""Weekly e2e coverage for BAGEL diffusion multi-replica serving. This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs. +It is covered by the weekly Diffusion · H100 · Multi-GPU lane +(``.buildkite/cuda/test-weekly.yml``, ``mirror_hardwares: h100_4``). """ import os @@ -17,7 +19,7 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT" MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml") ROUTE_STRESS_REQUESTS = 6 -pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.slow, pytest.mark.diffusion] test_params = [ OmniServerParams( diff --git a/tests/e2e/online_serving/test_boogu_image.py b/tests/e2e/online_serving/test_boogu_image.py index 5a5494d7d..440e031f1 100644 --- a/tests/e2e/online_serving/test_boogu_image.py +++ b/tests/e2e/online_serving/test_boogu_image.py @@ -16,7 +16,7 @@ parity vs. the Diffusers baseline is covered separately (see the support plan, s From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -28,7 +28,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Base" T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" diff --git a/tests/e2e/online_serving/test_boogu_image_edit.py b/tests/e2e/online_serving/test_boogu_image_edit.py index 45f999d26..d7b6d1dd1 100644 --- a/tests/e2e/online_serving/test_boogu_image_edit.py +++ b/tests/e2e/online_serving/test_boogu_image_edit.py @@ -19,7 +19,7 @@ reference image is supported. From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_edit.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import base64 @@ -35,7 +35,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] EDIT_PROMPT = "Change the style to a colored pencil drawing." SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) diff --git a/tests/e2e/online_serving/test_boogu_image_edit_expansion.py b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py index efc968eb6..1825aaafc 100644 --- a/tests/e2e/online_serving/test_boogu_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_boogu_image_edit_expansion.py @@ -25,7 +25,7 @@ model and are intentionally omitted. From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_edit_expansion.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -38,7 +38,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Edit" EDIT_PROMPT = "Change the style to a colored pencil drawing." diff --git a/tests/e2e/online_serving/test_boogu_image_expansion.py b/tests/e2e/online_serving/test_boogu_image_expansion.py index ef6d3a62e..a56e5a338 100644 --- a/tests/e2e/online_serving/test_boogu_image_expansion.py +++ b/tests/e2e/online_serving/test_boogu_image_expansion.py @@ -30,7 +30,7 @@ omitted (add rows here when those features land). From ``tests/``:: pytest -s -v e2e/online_serving/test_boogu_image_expansion.py \ - -m "full_model and diffusion and H100" --run-level=full_model + -m "slow and diffusion and H100" """ import os @@ -42,7 +42,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Boogu/Boogu-Image-0.1-Base" T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting" diff --git a/tests/e2e/online_serving/test_glm_image_expansion.py b/tests/e2e/online_serving/test_glm_image_expansion.py index d2a08f633..42e8d444a 100644 --- a/tests/e2e/online_serving/test_glm_image_expansion.py +++ b/tests/e2e/online_serving/test_glm_image_expansion.py @@ -30,7 +30,7 @@ from tests.helpers.runtime import ( ) from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image") diff --git a/tests/e2e/online_serving/test_lingbot_video_moe.py b/tests/e2e/online_serving/test_lingbot_video_moe.py index 777880732..7e4d16568 100644 --- a/tests/e2e/online_serving/test_lingbot_video_moe.py +++ b/tests/e2e/online_serving/test_lingbot_video_moe.py @@ -63,7 +63,7 @@ def test_text_to_image_moe(omni_server: OmniServer, openai_client: OpenAIClientH ) -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True) def test_video_generation_modes_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: diff --git a/tests/e2e/online_serving/test_ltx.py b/tests/e2e/online_serving/test_ltx.py index 43df5df2d..e4d8f51a8 100644 --- a/tests/e2e/online_serving/test_ltx.py +++ b/tests/e2e/online_serving/test_ltx.py @@ -10,7 +10,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] DISTILLED_MODEL = os.getenv("VLLM_TEST_LTX23_DISTILLED_MODEL", "diffusers/LTX-2.3-Distilled-Diffusers") BASE_MODEL = os.getenv("VLLM_TEST_LTX23_MODEL", "diffusers/LTX-2.3-Diffusers") diff --git a/tests/e2e/online_serving/test_ltx25.py b/tests/e2e/online_serving/test_ltx25.py index 2fad9f16a..da5c53851 100644 --- a/tests/e2e/online_serving/test_ltx25.py +++ b/tests/e2e/online_serving/test_ltx25.py @@ -20,7 +20,7 @@ MODEL = os.getenv("VLLM_TEST_LTX25_MODEL", DEFAULT_MODEL) MODEL_REVISION = os.getenv("VLLM_TEST_LTX25_MODEL_REVISION", DEFAULT_REVISION if MODEL == DEFAULT_MODEL else "") PROMPT = "A red fox walks through a snowy forest while the camera remains fixed." -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) diff --git a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py index 8ab327022..6f6881d68 100644 --- a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py +++ b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py @@ -20,7 +20,7 @@ from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -pytestmark = [pytest.mark.omni, pytest.mark.full_model] +pytestmark = [pytest.mark.omni, pytest.mark.slow] _SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip( reason="Requires 4x H100 GPUs; skipped in CI for now.", diff --git a/tests/e2e/online_serving/test_qwen_image_edit.py b/tests/e2e/online_serving/test_qwen_image_edit.py index a02bc1b57..14d27318c 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit.py +++ b/tests/e2e/online_serving/test_qwen_image_edit.py @@ -2,20 +2,19 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions). +Online serving tests for ``Qwen/Qwen-Image-Edit-2511`` (image-to-image via chat completions). -- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512. -- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512. -- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input - resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one - scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). +- ``test_single_image_to_image_001``: one reference image, fixed 512×512. +- ``test_multi_images_to_image_001``: two reference images, fixed 512×512. +- ``test_different_sizes_001``: ``slow`` — mixed input resolutions; ``extra_body`` uses + per-output ``width``/``height`` lists; the test client sends one scalar-size request + per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). -``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model. +``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row. From ``tests/``:: - pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model - pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model + pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "slow and diffusion" --run-level=full_model """ import pytest @@ -29,6 +28,7 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) +MODEL = "Qwen/Qwen-Image-Edit-2511" EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " @@ -39,7 +39,7 @@ NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) -def _get_diffusion_feature_cases(model: str): +def _get_diffusion_feature_cases(model: str = MODEL): """Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``).""" return [ pytest.param( @@ -56,11 +56,11 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"), + _get_diffusion_feature_cases(), indirect=True, ) def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1).""" + """Single-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511`` (CFG when negative prompt + ``true_cfg_scale`` > 1).""" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) @@ -86,7 +86,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): @@ -117,7 +117,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py index b3d6f8b19..e5e8c6c2b 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py @@ -1,8 +1,6 @@ """ Comprehensive tests of diffusion features that are available in online serving mode -and are supported by the following models: -- Qwen-Image-Edit: single image input -- Qwen-Image-Edit-2511: single image input and two image inputs +and are supported by Qwen-Image-Edit-2511 (single image input and two image inputs). """ import pytest @@ -11,10 +9,10 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] -EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." -SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting." +MODEL = "Qwen/Qwen-Image-Edit-2511" +SINGLE_EDIT_PROMPT = "Restyle this image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " "Transform the second image into a Vincent van Gogh style painting. " @@ -25,9 +23,7 @@ SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) -# This test file targets two models, so I write a helper function. -# If a similar test only involves one model, one can just define a global list variable. -def _get_diffusion_feature_cases(model: str): +def _get_diffusion_feature_cases(model: str = MODEL): return [ pytest.param( OmniServerParams( @@ -123,35 +119,7 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"), - indirect=True, -) -def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios.""" - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT) - - # CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "height": 512, - "width": 512, - "num_inference_steps": 2, - "negative_prompt": NEGATIVE_PROMPT, - "true_cfg_scale": 4.0, - "seed": 42, - }, - } - - openai_client.send_diffusion_request(request_config) - - -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler): @@ -166,7 +134,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511) + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT) request_config = { "model": omni_server.model, @@ -186,7 +154,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), + _get_diffusion_feature_cases(), indirect=True, ) def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py index 315fba6ef..8c5001a4a 100644 --- a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py @@ -18,7 +18,7 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import decode_b64_image, generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Qwen/Qwen-Image-Layered" EDIT_PROMPT = "Decompose this image into layers." diff --git a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py index 996585a1d..de73ae6da 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py @@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.full_model, pytest.mark.tts] +pytestmark = [pytest.mark.slow, pytest.mark.tts] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" diff --git a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py index d58bb9e32..4c6380314 100644 --- a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py +++ b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py @@ -26,7 +26,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] +pytestmark = [pytest.mark.diffusion, pytest.mark.slow] MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers" PROMPT = "A cat walking slowly across a sunlit garden path" diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py index 740f896f5..06f09c791 100644 --- a/tools/nightly/buildkite_testcase_statistics.py +++ b/tools/nightly/buildkite_testcase_statistics.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """ -Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml, -and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q -and produce an HTML report. +Parse pytest commands from Buildkite .buildkite/cuda/test-{ready,merge,nightly,weekly}.yml; +collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report. Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the group title carried into the report. @@ -30,7 +29,7 @@ import yaml # Repo root (parent of the directory containing this script) REPO_ROOT = Path(__file__).resolve().parent.parent.parent -BUILDKITE_DIR = REPO_ROOT / ".buildkite" +BUILDKITE_DIR = REPO_ROOT / ".buildkite" / "cuda" PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"] @@ -256,13 +255,34 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l rel_dir = Path(target.rstrip("/")) dir_path = (repo_root / rel_dir).resolve() if dir_path.exists() and dir_path.is_dir(): - return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120 + return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 300 if not extra: raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}") if not (repo_root / "tests").exists(): raise FileNotFoundError("tests/ directory not found under repo root") - return ["tests/"], "tests/", 120 + # Broad ``tests/`` collection loads many conftests; allow more time on busy CI agents. + return ["tests/"], "tests/", 300 + + +def _format_pytest_collect_failure( + path_args: list[str], + extra: list[str], + result: subprocess.CompletedProcess[str], +) -> str: + """pytest often puts collection errors on stdout; include both streams + exit code.""" + stdout = (result.stdout or "").strip() + stderr = (result.stderr or "").strip() + parts = [ + f"pytest --collect-only failed for {path_args} with args {extra} (exit={result.returncode})", + ] + if stderr: + parts.append(f"stderr:\n{stderr}") + if stdout: + parts.append(f"stdout:\n{stdout}") + if not stderr and not stdout: + parts.append("(no stdout/stderr captured)") + return "\n".join(parts) def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]: @@ -273,6 +293,8 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str] flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts parametrize from ``--test-config-file`` at import time). + + Pytest exit code 5 (no tests collected) is treated as an empty result, not a hard failure. """ extra = _pytest_collect_sidecar_args(raw_line) path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) @@ -286,8 +308,15 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str] text=True, timeout=timeout_quiet, ) + # 5 == no tests collected under the given -m/-k filters. + if result.returncode == 5: + print( + f"pytest --collect-only: no tests matched for {path_args} with args {extra}", + file=sys.stderr, + ) + return [] if result.returncode != 0: - raise RuntimeError(f"pytest --collect-only failed for {path_args} with args {extra}: {result.stderr.strip()}") + raise RuntimeError(_format_pytest_collect_failure(path_args, extra, result)) print(f"pytest --collect-only success for {path_args} with args {extra}: {result.stdout.strip()}") return _parse_collect_only_stdout(result.stdout or "", stderr=result.stderr or "") @@ -962,7 +991,7 @@ def main() -> None: "--buildkite-dir", type=Path, default=BUILDKITE_DIR, - help="Path to .buildkite directory", + help="Path to Buildkite CUDA pipeline directory (default: .buildkite/cuda)", ) default_out = REPO_ROOT / "buildkite_testcase_statistics.html" parser.add_argument( @@ -1023,7 +1052,12 @@ def main() -> None: names = [] skipped_ids: set[str] = set() for target, raw in targets: - collected = collect_test_names(target, REPO_ROOT, raw) + try: + collected = collect_test_names(target, REPO_ROOT, raw) + except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as exc: + # Keep generating the report for other steps; surface the root cause. + print(f"WARNING: collect failed for {module_key} target={target!r}: {exc}", file=sys.stderr) + collected = [] skipped_ids.update(get_skip_status(target, raw, REPO_ROOT)) names.extend(collected) count = len(names) - len(skipped_ids)