[CI]demote slow nightly cases and trim Qwen-Image-Edit coverage (#5944)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] <your.email@example.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
611c083587
commit
4616823125
35 files changed
+120
-341
No files matched your search
@@ -159,19 +159,7 @@ steps:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
|
||||
"
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
@@ -188,35 +176,6 @@ steps:
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: TTS · VoxCPM2 · Perf Test"
|
||||
key: nightly-tts-performance-voxcpm2
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
|
||||
key: nightly-tts-performance-higgs-audio-v3
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
|
||||
key: nightly-tiny-model-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
@@ -246,8 +205,7 @@ steps:
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
# Use explicit file shards so single-card cases do not reserve 2+ GPUs,
|
||||
# and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
|
||||
# Use explicit file shards so single-card cases do not reserve 2+ GPUs.
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
@@ -255,15 +213,6 @@ steps:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
|
||||
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
||||
tests/e2e/online_serving/test_boogu_image.py
|
||||
tests/e2e/online_serving/test_boogu_image_edit.py
|
||||
tests/e2e/online_serving/test_boogu_image_expansion.py
|
||||
tests/e2e/online_serving/test_boogu_image_edit_expansion.py
|
||||
tests/e2e/online_serving/test_bagel_expansion.py
|
||||
tests/e2e/online_serving/test_krea2_expansion.py
|
||||
tests/e2e/offline_inference/test_krea2_expansion.py
|
||||
-m "full_model and diffusion and H100 and not distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
@@ -274,47 +223,14 @@ steps:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
|
||||
tests/e2e/offline_inference/test_krea2_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_bagel_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_glm_image_expansion.py
|
||||
tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
|
||||
-m "full_model and diffusion and H100"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
|
||||
@@ -327,7 +243,7 @@ steps:
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
|
||||
- pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
@@ -355,70 +271,6 @@ steps:
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-layered
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
|
||||
key: nightly-diffusion-x2iat-performance-bagel
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
key: nightly-diffusion-x2v-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
@@ -440,12 +292,6 @@ steps:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Doc Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
@@ -456,15 +302,8 @@ steps:
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · LTX-2.5 Online Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_ltx25.py -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
@@ -483,23 +322,8 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · LTX Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model
|
||||
- pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test"
|
||||
key: nightly-diffusion-x2v-performance
|
||||
timeout_in_minutes: 180
|
||||
@@ -575,16 +399,6 @@ steps:
|
||||
- pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":bar_chart: Testcase Statistics"
|
||||
key: nightly-testcase-statistics
|
||||
timeout_in_minutes: 120
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
|
||||
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
|
||||
# Do not add new depends_on entries or extend this step.
|
||||
- label: ":email: Nightly Collection & Email"
|
||||
@@ -595,32 +409,20 @@ steps:
|
||||
- nightly-omni-performance-vllm-text
|
||||
- nightly-omni-performance-multi-replicas
|
||||
- nightly-tts-performance
|
||||
- nightly-tts-performance-higgs-audio-v3
|
||||
- nightly-diffusion-x2iat-performance-qwen-image
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-edit
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-layered
|
||||
- nightly-diffusion-x2iat-performance-bagel
|
||||
- nightly-diffusion-x2v-performance
|
||||
- nightly-testcase-statistics
|
||||
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
|
||||
commands:
|
||||
- pip install openpyxl
|
||||
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
|
||||
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
|
||||
- python tools/nightly/generate_nightly_perf_excel.py
|
||||
- python tools/nightly/generate_nightly_perf_html.py
|
||||
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
|
||||
|
||||
@@ -149,4 +149,4 @@ steps:
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_4
|
||||
@@ -96,33 +96,6 @@ steps:
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
||||
key: nightly-diffusion-x2iat-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_8
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion I2I Model Test"
|
||||
key: nightly-HunyuanImage3-x2i-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task",
|
||||
@@ -61,7 +61,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task",
|
||||
@@ -113,7 +113,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task",
|
||||
@@ -164,7 +164,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline with enforce-eager (no torch.compile)",
|
||||
@@ -64,7 +64,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device with torch.compile (default, no enforce-eager)",
|
||||
@@ -117,7 +117,7 @@
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with enforce-eager",
|
||||
@@ -172,7 +172,7 @@
|
||||
"num_cards": 2
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CFG-parallel=2 with torch.compile",
|
||||
@@ -226,7 +226,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "CacheDiT with enforce-eager",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline (two input images)",
|
||||
@@ -76,7 +76,7 @@
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
@@ -125,7 +125,7 @@
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
@@ -73,7 +73,7 @@
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
|
||||
@@ -121,7 +121,7 @@
|
||||
"num_cards": 4
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"diffusion"
|
||||
],
|
||||
"description": "Single-device baseline",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"local_model",
|
||||
"tts"
|
||||
],
|
||||
"server_params": {
|
||||
|
||||
@@ -463,7 +463,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@@ -591,7 +591,7 @@ def test_ltx25_distilled_two_stage_matches_official(accuracy_artifact_root: Path
|
||||
assert audio_metrics["relative_l2"] <= LTX25_AUDIO_RELATIVE_L2_THRESHOLD
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
|
||||
@@ -544,7 +544,7 @@ def _audio_metrics(reference: np.ndarray, prediction: np.ndarray) -> dict[str, f
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name)
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
|
||||
@@ -17,7 +17,7 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
pytestmark = [pytest.mark.diffusion]
|
||||
|
||||
|
||||
SINGLE_MODEL = "Qwen/Qwen-Image-Edit"
|
||||
@@ -253,6 +253,7 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_qwen_image_edit_single_matches_diffusers(
|
||||
@@ -278,6 +279,7 @@ def test_qwen_image_edit_single_matches_diffusers(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
|
||||
MODEL_ID = "Qwen/Qwen-Image-Layered"
|
||||
|
||||
@@ -11,7 +11,7 @@ from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_vi
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "BestWishYsh/Helios-Distilled"
|
||||
PROMPT = "A serene lakeside sunrise with mist over the water."
|
||||
|
||||
@@ -138,7 +138,7 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest:
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.slow,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
@@ -109,7 +109,7 @@ def _diffusion_sampling_params(
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.slow,
|
||||
pytest.mark.diffusion,
|
||||
]
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ def _get_test_config() -> str:
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/4700"),
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize(
|
||||
"omni_runner",
|
||||
@@ -271,7 +271,6 @@ def _collect_audio(omni_runner: OmniRunner, request: dict) -> tuple[torch.Tensor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str) -> None:
|
||||
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
|
||||
@@ -283,7 +282,6 @@ def test_moss_tts_realtime_english(omni_runner: OmniRunner, ref_audio_path: str)
|
||||
assert not torch.all(audio == 0), "Audio is silence"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_moss_tts_realtime_chinese(omni_runner: OmniRunner, ref_audio_path: str) -> None:
|
||||
"""MossTTSRealtime: Chinese input produces non-empty audio."""
|
||||
|
||||
@@ -22,7 +22,7 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution."
|
||||
NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark"
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving.
|
||||
"""Weekly e2e coverage for BAGEL diffusion multi-replica serving.
|
||||
|
||||
This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs.
|
||||
It is covered by the weekly Diffusion · H100 · Multi-GPU lane
|
||||
(``.buildkite/cuda/test-weekly.yml``, ``mirror_hardwares: h100_4``).
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -17,7 +19,7 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
|
||||
ROUTE_STRESS_REQUESTS = 6
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
test_params = [
|
||||
OmniServerParams(
|
||||
|
||||
@@ -16,7 +16,7 @@ parity vs. the Diffusers baseline is covered separately (see the support plan, s
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_boogu_image.py \
|
||||
-m "full_model and diffusion and H100" --run-level=full_model
|
||||
-m "slow and diffusion and H100"
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -28,7 +28,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "Boogu/Boogu-Image-0.1-Base"
|
||||
T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting"
|
||||
|
||||
@@ -19,7 +19,7 @@ reference image is supported.
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_boogu_image_edit.py \
|
||||
-m "full_model and diffusion and H100" --run-level=full_model
|
||||
-m "slow and diffusion and H100"
|
||||
"""
|
||||
|
||||
import base64
|
||||
@@ -35,7 +35,7 @@ from tests.helpers.runtime import (
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
EDIT_PROMPT = "Change the style to a colored pencil drawing."
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
@@ -25,7 +25,7 @@ model and are intentionally omitted.
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_boogu_image_edit_expansion.py \
|
||||
-m "full_model and diffusion and H100" --run-level=full_model
|
||||
-m "slow and diffusion and H100"
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -38,7 +38,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "Boogu/Boogu-Image-0.1-Edit"
|
||||
EDIT_PROMPT = "Change the style to a colored pencil drawing."
|
||||
|
||||
@@ -30,7 +30,7 @@ omitted (add rows here when those features land).
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_boogu_image_expansion.py \
|
||||
-m "full_model and diffusion and H100" --run-level=full_model
|
||||
-m "slow and diffusion and H100"
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -42,7 +42,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "Boogu/Boogu-Image-0.1-Base"
|
||||
T2I_PROMPT = "A mountain lake at sunset, photorealistic, cinematic lighting"
|
||||
|
||||
@@ -30,7 +30,7 @@ from tests.helpers.runtime import (
|
||||
)
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = os.environ.get("GLM_IMAGE_MODEL_PATH", "zai-org/GLM-Image")
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ def test_text_to_image_moe(omni_server: OmniServer, openai_client: OpenAIClientH
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True)
|
||||
def test_video_generation_modes_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
|
||||
@@ -10,7 +10,7 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
DISTILLED_MODEL = os.getenv("VLLM_TEST_LTX23_DISTILLED_MODEL", "diffusers/LTX-2.3-Distilled-Diffusers")
|
||||
BASE_MODEL = os.getenv("VLLM_TEST_LTX23_MODEL", "diffusers/LTX-2.3-Diffusers")
|
||||
|
||||
@@ -20,7 +20,7 @@ MODEL = os.getenv("VLLM_TEST_LTX25_MODEL", DEFAULT_MODEL)
|
||||
MODEL_REVISION = os.getenv("VLLM_TEST_LTX25_MODEL_REVISION", DEFAULT_REVISION if MODEL == DEFAULT_MODEL else "")
|
||||
PROMPT = "A red fox walks through a snowy forest while the camera remains fixed."
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.slow]
|
||||
|
||||
_SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip(
|
||||
reason="Requires 4x H100 GPUs; skipped in CI for now.",
|
||||
|
||||
@@ -2,20 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
|
||||
Online serving tests for ``Qwen/Qwen-Image-Edit-2511`` (image-to-image via chat completions).
|
||||
|
||||
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
|
||||
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input
|
||||
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
|
||||
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
|
||||
- ``test_single_image_to_image_001``: one reference image, fixed 512×512.
|
||||
- ``test_multi_images_to_image_001``: two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``slow`` — mixed input resolutions; ``extra_body`` uses
|
||||
per-output ``width``/``height`` lists; the test client sends one scalar-size request
|
||||
per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
|
||||
|
||||
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model.
|
||||
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row.
|
||||
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model
|
||||
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model
|
||||
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "slow and diffusion" --run-level=full_model
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -29,6 +28,7 @@ from tests.helpers.runtime import (
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
MODEL = "Qwen/Qwen-Image-Edit-2511"
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
@@ -39,7 +39,7 @@ NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
def _get_diffusion_feature_cases(model: str = MODEL):
|
||||
"""Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``)."""
|
||||
return [
|
||||
pytest.param(
|
||||
@@ -56,11 +56,11 @@ def _get_diffusion_feature_cases(model: str):
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
|
||||
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
@@ -86,7 +86,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
@@ -117,7 +117,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""
|
||||
Comprehensive tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following models:
|
||||
- Qwen-Image-Edit: single image input
|
||||
- Qwen-Image-Edit-2511: single image input and two image inputs
|
||||
and are supported by Qwen-Image-Edit-2511 (single image input and two image inputs).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -11,10 +9,10 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting."
|
||||
MODEL = "Qwen/Qwen-Image-Edit-2511"
|
||||
SINGLE_EDIT_PROMPT = "Restyle this image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
@@ -25,9 +23,7 @@ SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||
|
||||
|
||||
# This test file targets two models, so I write a helper function.
|
||||
# If a similar test only involves one model, one can just define a global list variable.
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
def _get_diffusion_feature_cases(model: str = MODEL):
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
@@ -123,35 +119,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
@@ -166,7 +134,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
|
||||
"""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511)
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
@@ -186,7 +154,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
|
||||
@@ -18,7 +18,7 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import decode_b64_image, generate_synthetic_image
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "Qwen/Qwen-Image-Layered"
|
||||
EDIT_PROMPT = "Decompose this image into layers."
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import pytest
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers"
|
||||
PROMPT = "A cat walking slowly across a sunlit garden path"
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml,
|
||||
and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q
|
||||
and produce an HTML report.
|
||||
Parse pytest commands from Buildkite .buildkite/cuda/test-{ready,merge,nightly,weekly}.yml;
|
||||
collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report.
|
||||
|
||||
Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the
|
||||
group title carried into the report.
|
||||
@@ -30,7 +29,7 @@ import yaml
|
||||
|
||||
# Repo root (parent of the directory containing this script)
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
BUILDKITE_DIR = REPO_ROOT / ".buildkite"
|
||||
BUILDKITE_DIR = REPO_ROOT / ".buildkite" / "cuda"
|
||||
PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"]
|
||||
|
||||
|
||||
@@ -256,13 +255,34 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l
|
||||
rel_dir = Path(target.rstrip("/"))
|
||||
dir_path = (repo_root / rel_dir).resolve()
|
||||
if dir_path.exists() and dir_path.is_dir():
|
||||
return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120
|
||||
return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 300
|
||||
|
||||
if not extra:
|
||||
raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}")
|
||||
if not (repo_root / "tests").exists():
|
||||
raise FileNotFoundError("tests/ directory not found under repo root")
|
||||
return ["tests/"], "tests/", 120
|
||||
# Broad ``tests/`` collection loads many conftests; allow more time on busy CI agents.
|
||||
return ["tests/"], "tests/", 300
|
||||
|
||||
|
||||
def _format_pytest_collect_failure(
|
||||
path_args: list[str],
|
||||
extra: list[str],
|
||||
result: subprocess.CompletedProcess[str],
|
||||
) -> str:
|
||||
"""pytest often puts collection errors on stdout; include both streams + exit code."""
|
||||
stdout = (result.stdout or "").strip()
|
||||
stderr = (result.stderr or "").strip()
|
||||
parts = [
|
||||
f"pytest --collect-only failed for {path_args} with args {extra} (exit={result.returncode})",
|
||||
]
|
||||
if stderr:
|
||||
parts.append(f"stderr:\n{stderr}")
|
||||
if stdout:
|
||||
parts.append(f"stdout:\n{stdout}")
|
||||
if not stderr and not stdout:
|
||||
parts.append("(no stdout/stderr captured)")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]:
|
||||
@@ -273,6 +293,8 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]
|
||||
flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the
|
||||
full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts
|
||||
parametrize from ``--test-config-file`` at import time).
|
||||
|
||||
Pytest exit code 5 (no tests collected) is treated as an empty result, not a hard failure.
|
||||
"""
|
||||
extra = _pytest_collect_sidecar_args(raw_line)
|
||||
path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line)
|
||||
@@ -286,8 +308,15 @@ def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]
|
||||
text=True,
|
||||
timeout=timeout_quiet,
|
||||
)
|
||||
# 5 == no tests collected under the given -m/-k filters.
|
||||
if result.returncode == 5:
|
||||
print(
|
||||
f"pytest --collect-only: no tests matched for {path_args} with args {extra}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return []
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pytest --collect-only failed for {path_args} with args {extra}: {result.stderr.strip()}")
|
||||
raise RuntimeError(_format_pytest_collect_failure(path_args, extra, result))
|
||||
print(f"pytest --collect-only success for {path_args} with args {extra}: {result.stdout.strip()}")
|
||||
return _parse_collect_only_stdout(result.stdout or "", stderr=result.stderr or "")
|
||||
|
||||
@@ -962,7 +991,7 @@ def main() -> None:
|
||||
"--buildkite-dir",
|
||||
type=Path,
|
||||
default=BUILDKITE_DIR,
|
||||
help="Path to .buildkite directory",
|
||||
help="Path to Buildkite CUDA pipeline directory (default: .buildkite/cuda)",
|
||||
)
|
||||
default_out = REPO_ROOT / "buildkite_testcase_statistics.html"
|
||||
parser.add_argument(
|
||||
@@ -1023,7 +1052,12 @@ def main() -> None:
|
||||
names = []
|
||||
skipped_ids: set[str] = set()
|
||||
for target, raw in targets:
|
||||
collected = collect_test_names(target, REPO_ROOT, raw)
|
||||
try:
|
||||
collected = collect_test_names(target, REPO_ROOT, raw)
|
||||
except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as exc:
|
||||
# Keep generating the report for other steps; surface the root cause.
|
||||
print(f"WARNING: collect failed for {module_key} target={target!r}: {exc}", file=sys.stderr)
|
||||
collected = []
|
||||
skipped_ids.update(get_skip_status(target, raw, REPO_ROOT))
|
||||
names.extend(collected)
|
||||
count = len(names) - len(skipped_ids)
|
||||
|
||||
Reference in new issue
Block a user