[CI]demote slow nightly cases and trim Qwen-Image-Edit coverage (#5944)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wangyuandCursor authored and GitHub committed 2026-08-17 11:21:28 +08:00
1 parent 611c083587
commit 4616823125
35 files changed
+120 -341

No files matched your search

+4 -202
View File
@@ -159,19 +159,7 @@ steps:
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- export VLLM_USE_DEEP_GEMM="0"
- export VLLM_MOE_USE_DEEP_GEMM="0"
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
mirror_hardwares: l4_1
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
timeout_in_minutes: 30
commands:
- |
timeout 30m bash -c "
export VLLM_LOGGING_LEVEL=DEBUG
export VLLM_WORKER_MULTIPROC_METHOD=spawn
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
"
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
mirror_hardwares: l4_1
- label: ":full_moon: TTS · Perf Test"
@@ -188,35 +176,6 @@ steps:
exit $$EXIT
mirror_hardwares: h100_2
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
key: nightly-tts-performance-voxcpm2
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
mirror_hardwares: h100_1
- label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
key: nightly-tts-performance-higgs-audio-v3
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- export VLLM_USE_DEEP_GEMM="0"
- export VLLM_MOE_USE_DEEP_GEMM="0"
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
mirror_hardwares: h100_1
- group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
key: nightly-tiny-model-test-group
depends_on: upload-nightly-pipeline
@@ -246,8 +205,7 @@ steps:
build.pull_request.labels includes "nightly-test" ||
build.pull_request.labels includes "diffusion-x2iat-test"
steps:
# Use explicit file shards so single-card cases do not reserve 2+ GPUs,
# and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
# Use explicit file shards so single-card cases do not reserve 2+ GPUs.
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
timeout_in_minutes: 120
@@ -255,15 +213,6 @@ steps:
- >-
pytest -sv
tests/e2e/online_serving/test_qwen_image_expansion.py
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
tests/e2e/online_serving/test_boogu_image.py
tests/e2e/online_serving/test_boogu_image_edit.py
tests/e2e/online_serving/test_boogu_image_expansion.py
tests/e2e/online_serving/test_boogu_image_edit_expansion.py
tests/e2e/online_serving/test_bagel_expansion.py
tests/e2e/online_serving/test_krea2_expansion.py
tests/e2e/offline_inference/test_krea2_expansion.py
-m "full_model and diffusion and H100 and not distributed_cuda"
--run-level "full_model"
mirror_hardwares: h100_1
@@ -274,47 +223,14 @@ steps:
- >-
pytest -sv
tests/e2e/online_serving/test_qwen_image_expansion.py
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
tests/e2e/offline_inference/test_krea2_expansion.py
-m "full_model and diffusion and H100 and distributed_cuda"
--run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
timeout_in_minutes: 90
commands:
- >-
pytest -sv
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
-m "full_model and diffusion and H100 and distributed_cuda"
--run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
timeout_in_minutes: 120
commands:
- >-
pytest -sv
tests/e2e/online_serving/test_bagel_expansion.py
-m "full_model and diffusion and H100 and distributed_cuda"
--run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
timeout_in_minutes: 120
commands:
- >-
pytest -sv
tests/e2e/online_serving/test_glm_image_expansion.py
tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
-m "full_model and diffusion and H100"
--run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
timeout_in_minutes: 120
commands:
- pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
mirror_hardwares: l4_4
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -327,7 +243,7 @@ steps:
timeout_in_minutes: 180
commands:
- export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
- pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
- pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
mirror_hardwares: h100_1
@@ -355,70 +271,6 @@ steps:
exit $$EXIT
mirror_hardwares: h100_4
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
key: nightly-diffusion-x2iat-performance-qwen-image-edit
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_4
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_4
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
key: nightly-diffusion-x2iat-performance-qwen-image-layered
timeout_in_minutes: 120
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
key: nightly-diffusion-x2iat-performance-bagel
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_1
- group: ":card_index_dividers: Diffusion X2V Model Test"
key: nightly-diffusion-x2v-group
depends_on: upload-nightly-pipeline
@@ -440,12 +292,6 @@ steps:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Doc Test"
timeout_in_minutes: 60
commands:
@@ -456,15 +302,8 @@ steps:
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
- pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · LTX-2.5 Online Test"
timeout_in_minutes: 60
commands:
- pytest -s -v tests/e2e/online_serving/test_ltx25.py -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
timeout_in_minutes: 180
commands:
@@ -483,23 +322,8 @@ steps:
commands:
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
- pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · LTX Accuracy Test"
timeout_in_minutes: 180
commands:
- pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model
- pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test"
timeout_in_minutes: 180
commands:
- pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model
- pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Perf Test"
key: nightly-diffusion-x2v-performance
timeout_in_minutes: 180
@@ -575,16 +399,6 @@ steps:
- pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
mirror_hardwares: l4_1
- label: ":bar_chart: Testcase Statistics"
key: nightly-testcase-statistics
timeout_in_minutes: 120
depends_on: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
- python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
mirror_hardwares: h100_1
# NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
# Do not add new depends_on entries or extend this step.
- label: ":email: Nightly Collection & Email"
@@ -595,32 +409,20 @@ steps:
- nightly-omni-performance-vllm-text
- nightly-omni-performance-multi-replicas
- nightly-tts-performance
- nightly-tts-performance-higgs-audio-v3
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2iat-performance-qwen-image-edit
- nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- nightly-diffusion-x2iat-performance-qwen-image-layered
- nightly-diffusion-x2iat-performance-bagel
- nightly-diffusion-x2v-performance
- nightly-testcase-statistics
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
commands:
- pip install openpyxl
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
- python tools/nightly/generate_nightly_perf_excel.py
- python tools/nightly/generate_nightly_perf_html.py
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
+1 -1
View File
@@ -149,4 +149,4 @@ steps:
pytest -sv
tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
--run-level "full_model"
mirror_hardwares: h100_2
mirror_hardwares: h100_4
-27
View File
@@ -96,33 +96,6 @@ steps:
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
key: nightly-diffusion-x2iat-group
depends_on: upload-nightly-pipeline
if: >-
build.env("NIGHTLY") == "1" ||
build.pull_request.labels includes "nightly-test" ||
build.pull_request.labels includes "diffusion-x2iat-test"
steps:
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
timeout_in_minutes: 180
mirror_hardwares: a3_npu_8
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
HF_TOKEN: "${HF_TOKEN}"
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
- group: ":card_index_dividers: Diffusion I2I Model Test"
key: nightly-HunyuanImage3-x2i-test-group
depends_on: upload-nightly-pipeline
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task",
@@ -61,7 +61,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task",
@@ -113,7 +113,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task",
@@ -164,7 +164,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task",
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"tts"
],
"server_params": {
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-device baseline with enforce-eager (no torch.compile)",
@@ -64,7 +64,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-device with torch.compile (default, no enforce-eager)",
@@ -117,7 +117,7 @@
"num_cards": 2
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "CFG-parallel=2 with enforce-eager",
@@ -172,7 +172,7 @@
"num_cards": 2
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "CFG-parallel=2 with torch.compile",
@@ -226,7 +226,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "CacheDiT with enforce-eager",
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-device baseline (two input images)",
@@ -76,7 +76,7 @@
"num_cards": 4
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
@@ -125,7 +125,7 @@
"num_cards": 4
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-device baseline",
@@ -73,7 +73,7 @@
"num_cards": 4
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
@@ -121,7 +121,7 @@
"num_cards": 4
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"diffusion"
],
"description": "Single-device baseline",
+1 -1
View File
@@ -10,7 +10,7 @@
"num_cards": 1
}
},
"full_model",
"local_model",
"tts"
],
"server_params": {
@@ -463,7 +463,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f
}
@pytest.mark.full_model
@pytest.mark.slow
@pytest.mark.benchmark
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@@ -591,7 +591,7 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path
assert audio_metrics["relative_l2"] <= LTX25_AUDIO_RELATIVE_L2_THRESHOLD
@pytest.mark.full_model
@pytest.mark.slow
@pytest.mark.benchmark
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@@ -544,7 +544,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f
@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name)
@pytest.mark.full_model
@pytest.mark.slow
@pytest.mark.benchmark
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100"}, num_cards=1)
+3 -1
View File
@@ -17,7 +17,7 @@ from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
pytestmark = [pytest.mark.diffusion]
SINGLE_MODEL = "Qwen/Qwen-Image-Edit"
@@ -253,6 +253,7 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image
)
@pytest.mark.slow
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_single_matches_diffusers(
@@ -278,6 +279,7 @@ def test_qwen_image_edit_single_matches_diffusers(
)
@pytest.mark.slow
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
@@ -17,7 +17,7 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
MODEL_ID = "Qwen/Qwen-Image-Layered"
@@ -11,7 +11,7 @@ from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_vi
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "BestWishYsh/Helios-Distilled"
PROMPT = "A serene lakeside sunrise with mist over the water."
@@ -138,7 +138,7 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest:
pytestmark = [
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
@@ -109,7 +109,7 @@ def _diffusion_sampling_params(
pytestmark = [
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.diffusion,
]
@@ -96,7 +96,7 @@ def _get_test_config() -> str:
pytestmark = [
pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"),
pytest.mark.full_model,
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.parametrize(
"omni_runner",
@@ -271,7 +271,6 @@ def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor
# ---------------------------------------------------------------------------
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) -> None:
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
@@ -283,7 +282,6 @@ def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str)
assert not torch.all(audio == 0), "Audio is silence"
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_moss_tts_realtime_chinese(omni_runner: OmniRunner, ref_audio_path: str) -> None:
"""MossTTSRealtime: Chinese input produces non-empty audio."""
@@ -22,7 +22,7 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution."
NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark"
@@ -1,6 +1,8 @@
"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving.
"""Weekly e2e coverage for BAGEL diffusion multi-replica serving.
This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs.
It is covered by the weekly Diffusion · H100 · Multi-GPU lane
(``.buildkite/cuda/test-weekly.yml``, ``mirror_hardwares: h100_4``).
"""
import os
@@ -17,7 +19,7 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
ROUTE_STRESS_REQUESTS = 6
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
test_params = [
OmniServerParams(
+2 -2
View File
@@ -16,7 +16,7 @@ parity vs. the Diffusers baseline is covered separately (see the support plan, s
From ``tests/``::
pytest -s -v e2e/online_serving/test_boogu_image.py \
-m "full_model and diffusion and H100" --run-level=full_model
-m "slow and diffusion and H100"
"""
import os
@@ -28,7 +28,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "Boogu/Boogu-Image-0.1-Base"
T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting"
@@ -19,7 +19,7 @@ reference image is supported.
From ``tests/``::
pytest -s -v e2e/online_serving/test_boogu_image_edit.py \
-m "full_model and diffusion and H100" --run-level=full_model
-m "slow and diffusion and H100"
"""
import base64
@@ -35,7 +35,7 @@ from tests.helpers.runtime import (
dummy_messages_from_mix_data,
)
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
EDIT_PROMPT = "Change the style to a colored pencil drawing."
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
@@ -25,7 +25,7 @@ model and are intentionally omitted.
From ``tests/``::
pytest -s -v e2e/online_serving/test_boogu_image_edit_expansion.py \
-m "full_model and diffusion and H100" --run-level=full_model
-m "slow and diffusion and H100"
"""
import os
@@ -38,7 +38,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "Boogu/Boogu-Image-0.1-Edit"
EDIT_PROMPT = "Change the style to a colored pencil drawing."
@@ -30,7 +30,7 @@ omitted (add rows here when those features land).
From ``tests/``::
pytest -s -v e2e/online_serving/test_boogu_image_expansion.py \
-m "full_model and diffusion and H100" --run-level=full_model
-m "slow and diffusion and H100"
"""
import os
@@ -42,7 +42,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "Boogu/Boogu-Image-0.1-Base"
T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting"
@@ -30,7 +30,7 @@ from tests.helpers.runtime import (
)
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
@@ -63,7 +63,7 @@ def test_text_to_image_moe(omni_server: OmniServer, openai_client: OpenAIClientH
)
@pytest.mark.full_model
@pytest.mark.slow
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True)
def test_video_generation_modes_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+1 -1
View File
@@ -10,7 +10,7 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
DISTILLED_MODEL = os.getenv("VLLM_TEST_LTX23_DISTILLED_MODEL", "diffusers/LTX-2.3-Distilled-Diffusers")
BASE_MODEL = os.getenv("VLLM_TEST_LTX23_MODEL", "diffusers/LTX-2.3-Diffusers")
+1 -1
View File
@@ -20,7 +20,7 @@ MODEL = os.getenv("VLLM_TEST_LTX25_MODEL", DEFAULT_MODEL)
MODEL_REVISION = os.getenv("VLLM_TEST_LTX25_MODEL_REVISION", DEFAULT_REVISION if MODEL == DEFAULT_MODEL else "")
PROMPT = "A red fox walks through a snowy forest while the camera remains fixed."
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
@@ -20,7 +20,7 @@ from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
pytestmark = [pytest.mark.omni, pytest.mark.slow]
_SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip(
reason="Requires 4x H100 GPUs; skipped in CI for now.",
@@ -2,20 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
Online serving tests for ``Qwen/Qwen-Image-Edit-2511`` (image-to-image via chat completions).
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
- ``test_single_image_to_image_001``: one reference image, fixed 512×512.
- ``test_multi_images_to_image_001``: two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``slow`` — mixed input resolutions; ``extra_body`` uses
per-output ``width``/``height`` lists; the test client sends one scalar-size request
per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model.
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row.
From ``tests/``::
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "slow and diffusion" --run-level=full_model
"""
import pytest
@@ -29,6 +28,7 @@ from tests.helpers.runtime import (
dummy_messages_from_mix_data,
)
MODEL = "Qwen/Qwen-Image-Edit-2511"
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
@@ -39,7 +39,7 @@ NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
def _get_diffusion_feature_cases(model: str = MODEL):
"""Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``)."""
return [
pytest.param(
@@ -56,11 +56,11 @@ def _get_diffusion_feature_cases(model: str):
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
_get_diffusion_feature_cases(),
indirect=True,
)
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
@@ -86,7 +86,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
_get_diffusion_feature_cases(),
indirect=True,
)
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
@@ -117,7 +117,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
_get_diffusion_feature_cases(),
indirect=True,
)
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
@@ -1,8 +1,6 @@
"""
Comprehensive tests of diffusion features that are available in online serving mode
and are supported by the following models:
- Qwen-Image-Edit: single image input
- Qwen-Image-Edit-2511: single image input and two image inputs
and are supported by Qwen-Image-Edit-2511 (single image input and two image inputs).
"""
import pytest
@@ -11,10 +9,10 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting."
MODEL = "Qwen/Qwen-Image-Edit-2511"
SINGLE_EDIT_PROMPT = "Restyle this image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
@@ -25,9 +23,7 @@ SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
# This test file targets two models, so I write a helper function.
# If a similar test only involves one model, one can just define a global list variable.
def _get_diffusion_feature_cases(model: str):
def _get_diffusion_feature_cases(model: str = MODEL):
return [
pytest.param(
OmniServerParams(
@@ -123,35 +119,7 @@ def _get_diffusion_feature_cases(model: str):
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
indirect=True,
)
def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
_get_diffusion_feature_cases(),
indirect=True,
)
def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
@@ -166,7 +134,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
"""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511)
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT)
request_config = {
"model": omni_server.model,
@@ -186,7 +154,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
_get_diffusion_feature_cases(),
indirect=True,
)
def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler):
@@ -18,7 +18,7 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.media import decode_b64_image, generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "Qwen/Qwen-Image-Layered"
EDIT_PROMPT = "Decompose this image into layers."
@@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
pytestmark = [pytest.mark.slow, pytest.mark.tts]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -26,7 +26,7 @@ import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers"
PROMPT = "A cat walking slowly across a sunlit garden path"
+43 -9
View File
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
"""
Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml,
and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q
and produce an HTML report.
Parse pytest commands from Buildkite .buildkite/cuda/test-{ready,merge,nightly,weekly}.yml;
collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report.
Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the
group title carried into the report.
@@ -30,7 +29,7 @@ import yaml
# Repo root (parent of the directory containing this script)
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
BUILDKITE_DIR = REPO_ROOT / ".buildkite"
BUILDKITE_DIR = REPO_ROOT / ".buildkite" / "cuda"
PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"]
@@ -256,13 +255,34 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l
rel_dir = Path(target.rstrip("/"))
dir_path = (repo_root / rel_dir).resolve()
if dir_path.exists() and dir_path.is_dir():
return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120
return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 300
if not extra:
raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}")
if not (repo_root / "tests").exists():
raise FileNotFoundError("tests/ directory not found under repo root")
return ["tests/"], "tests/", 120
# Broad ``tests/`` collection loads many conftests; allow more time on busy CI agents.
return ["tests/"], "tests/", 300
def _format_pytest_collect_failure(
path_args: list[str],
extra: list[str],
result: subprocess.CompletedProcess[str],
) -> str:
"""pytest often puts collection errors on stdout; include both streams + exit code."""
stdout = (result.stdout or "").strip()
stderr = (result.stderr or "").strip()
parts = [
f"pytest --collect-only failed for {path_args} with args {extra} (exit={result.returncode})",
]
if stderr:
parts.append(f"stderr:\n{stderr}")
if stdout:
parts.append(f"stdout:\n{stdout}")
if not stderr and not stdout:
parts.append("(no stdout/stderr captured)")
return "\n".join(parts)
def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]:
@@ -273,6 +293,8 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]
flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the
full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts
parametrize from ``--test-config-file`` at import time).
Pytest exit code 5 (no tests collected) is treated as an empty result, not a hard failure.
"""
extra = _pytest_collect_sidecar_args(raw_line)
path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line)
@@ -286,8 +308,15 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]
text=True,
timeout=timeout_quiet,
)
# 5 == no tests collected under the given -m/-k filters.
if result.returncode == 5:
print(
f"pytest --collect-only: no tests matched for {path_args} with args {extra}",
file=sys.stderr,
)
return []
if result.returncode != 0:
raise RuntimeError(f"pytest --collect-only failed for {path_args} with args {extra}: {result.stderr.strip()}")
raise RuntimeError(_format_pytest_collect_failure(path_args, extra, result))
print(f"pytest --collect-only success for {path_args} with args {extra}: {result.stdout.strip()}")
return _parse_collect_only_stdout(result.stdout or "", stderr=result.stderr or "")
@@ -962,7 +991,7 @@ def main() -> None:
"--buildkite-dir",
type=Path,
default=BUILDKITE_DIR,
help="Path to .buildkite directory",
help="Path to Buildkite CUDA pipeline directory (default: .buildkite/cuda)",
)
default_out = REPO_ROOT / "buildkite_testcase_statistics.html"
parser.add_argument(
@@ -1023,7 +1052,12 @@ def main() -> None:
names = []
skipped_ids: set[str] = set()
for target, raw in targets:
collected = collect_test_names(target, REPO_ROOT, raw)
try:
collected = collect_test_names(target, REPO_ROOT, raw)
except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as exc:
# Keep generating the report for other steps; surface the root cause.
print(f"WARNING: collect failed for {module_key} target={target!r}: {exc}", file=sys.stderr)
collected = []
skipped_ids.update(get_skip_status(target, raw, REPO_ROOT))
names.extend(collected)
count = len(names) - len(skipped_ids)