From d671e6b0db1fe93d59efa23b435d51f0215f779e Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:01:20 +0800 Subject: [PATCH] [CI] Expand E2E source_file_dependencies for shared model code (#5994) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/cuda/test-merge.yml | 117 ++++-------------- .buildkite/cuda/test-ready.yml | 103 +++++---------- docs/contributing/ci/ci_settings.md | 15 ++- tests/buildkite/test_upload_pipeline.py | 57 +++++---- tests/e2e/offline_inference/test_bagel.py | 2 +- .../e2e/offline_inference/test_flux2_klein.py | 4 +- .../e2e/offline_inference/test_voxcpm2_tts.py | 9 +- tests/e2e/offline_inference/test_z_image.py | 2 +- tests/e2e/online_serving/test_bagel.py | 6 +- .../e2e/online_serving/test_higgs_audio_v3.py | 27 ++-- .../online_serving/test_qwen_image_edit.py | 9 +- .../online_serving/test_qwen_image_layered.py | 3 +- tests/e2e/online_serving/test_voxcpm2_tts.py | 5 +- tests/e2e/online_serving/test_z_image.py | 4 +- 14 files changed, 135 insertions(+), 228 deletions(-) diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 970ab5a5a..69913bf55 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -76,6 +76,9 @@ steps: - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -92,6 +95,9 @@ steps: - tests/e2e/online_serving/test_qwen3_tts_base.py - tests/e2e/offline_inference/test_qwen3_tts_base.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml - .buildkite/common/scripts/run_cov_split.sh @@ -109,27 +115,15 @@ steps: -- -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 - - label: "Omni · Qwen3-Omni Test" source_file_dependencies: - tests/e2e/offline_inference/test_qwen3_omni.py - tests/e2e/online_serving/test_qwen3_omni.py - vllm_omni/model_executor/models/qwen3_omni/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py + - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml timeout_in_minutes: 50 @@ -141,6 +135,11 @@ steps: source_file_dependencies: - tests/e2e/offline_inference/test_minicpmo_4_5.py - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml timeout_in_minutes: 50 commands: @@ -162,70 +161,35 @@ steps: - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/core/sched/ - vllm_omni/outputs/multimodal_accumulation.py - vllm_omni/engine/stage_init_utils.py - vllm_omni/engine/async_omni_engine.py + - vllm_omni/engine/orchestrator.py - vllm_omni/entrypoints/cli/serve.py + - vllm_omni/entrypoints/openai/api_server.py + - vllm_omni/entrypoints/openai/duplex_capability.py + - vllm_omni/config/stage_config.py timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_1 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_voxcpm2_tts.py - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 - - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_bagel.py - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - - .buildkite/common/scripts/run_cov_split.sh - - pyproject.toml - # 20 -> 40: the coverage split runs the model twice. - timeout_in_minutes: 40 - commands: - - | - .buildkite/common/scripts/run_cov_split.sh \ - --model-id bagel \ - --offline tests/e2e/offline_inference/test_bagel.py \ - --online tests/e2e/online_serving/test_bagel.py \ - -- -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen_image.py - vllm_omni/diffusion/models/qwen_image/ + - vllm_omni/diffusion/models/dmd2/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py commands: - | timeout 40m bash -c ' @@ -234,23 +198,12 @@ steps: ' mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" source_file_dependencies: - tests/e2e/offline_inference/test_wan22_t2v.py - tests/e2e/online_serving/test_wan22_t2v.py - vllm_omni/diffusion/models/wan2_2/ + - vllm_omni/diffusion/models/dmd2/ commands: - | timeout 40m bash -c " @@ -258,21 +211,3 @@ steps: pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' " mirror_hardwares: h100_1 - - - label: "Diffusion · Z Image Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_z_image.py - - tests/e2e/online_serving/test_z_image.py - - vllm_omni/diffusion/models/z_image/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 - - label: "Diffusion · FLUX.2 Klein Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_flux2_klein.py - - vllm_omni/diffusion/models/flux2_klein/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index f416631ff..8dbe7bd71 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -169,6 +169,10 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen3_omni.py - vllm_omni/model_executor/models/qwen3_omni/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py + - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - vllm_omni/deploy/qwen3_omni_moe.yaml commands: @@ -181,6 +185,11 @@ steps: - label: "Omni · MiniCPM-o 4.5 Offline Test" source_file_dependencies: - tests/e2e/offline_inference/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | @@ -192,6 +201,11 @@ steps: - label: "Omni · MiniCPM-o 4.5 Online Test" source_file_dependencies: - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | @@ -210,16 +224,24 @@ steps: - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py + - vllm_omni/deploy/minicpmo_4_5.yaml - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - vllm_omni/experimental/fullduplex/ - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/models/step_audio2/ + - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ + - vllm_omni/model_executor/models/cosyvoice3/utils.py - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - vllm_omni/core/sched/ - vllm_omni/outputs/multimodal_accumulation.py - vllm_omni/engine/stage_init_utils.py - vllm_omni/engine/async_omni_engine.py + - vllm_omni/engine/orchestrator.py - vllm_omni/entrypoints/cli/serve.py + - vllm_omni/entrypoints/openai/api_server.py + - vllm_omni/entrypoints/openai/duplex_capability.py + - vllm_omni/config/stage_config.py commands: - | timeout 20m bash -c " @@ -239,25 +261,13 @@ steps: - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda' mirror_hardwares: h100_1 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 - - label: "TTS · Qwen3-TTS CustomVoice Test" source_file_dependencies: - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -272,6 +282,9 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen3_tts_base.py - vllm_omni/model_executor/models/qwen3_tts/ + - vllm_omni/model_executor/models/common/qwen3_code_predictor.py + - vllm_omni/model_executor/models/common/snake_activation.py + - vllm_omni/model_executor/models/common/alias_free_activation.py - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - vllm_omni/deploy/qwen3_tts.yaml commands: @@ -284,52 +297,11 @@ steps: " mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 - - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 20m bash -c " - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 - - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" source_file_dependencies: - tests/e2e/online_serving/test_wan22_t2v.py - vllm_omni/diffusion/models/wan2_2/ + - vllm_omni/diffusion/models/dmd2/ commands: - | timeout 40m bash -c " @@ -342,6 +314,9 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_cosmos3.py - vllm_omni/diffusion/models/cosmos3/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py + - vllm_omni/experimental/world_models/ + - vllm_omni/model_extras/cosmos3.py commands: - | timeout 40m bash -c " @@ -354,6 +329,8 @@ steps: source_file_dependencies: - tests/e2e/online_serving/test_qwen_image.py - vllm_omni/diffusion/models/qwen_image/ + - vllm_omni/diffusion/models/dmd2/ + - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py commands: - | timeout 40m bash -c " @@ -361,15 +338,3 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' " mirror_hardwares: h100_1 - - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md index 62717ddc9..035e8c650 100644 --- a/docs/contributing/ci/ci_settings.md +++ b/docs/contributing/ci/ci_settings.md @@ -329,11 +329,13 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for ### Per-model coverage -Pilot (v1): two Merge-tier (L3) jobs in `.buildkite/cuda/test-merge.yml` upload -per-model, per-entry-mode coverage as Buildkite artifacts — "Diffusion · Bagel -Test" and "TTS · Qwen3-TTS Base Test". Both run on a single GPU so the pilot is -cheap to reproduce. This is a pilot, not a rollout: no dashboard/visualization -lives in this repo, and nothing is gated on coverage. +Pilot (v1): one Merge-tier (L3) job in `.buildkite/cuda/test-merge.yml` uploads +per-model, per-entry-mode coverage as Buildkite artifacts — "TTS · Qwen3-TTS Base +Test". It runs on a single GPU so the pilot is cheap to reproduce. (A second +diffusion pilot, "Diffusion · Bagel Test", was retired from merge; reintroduce +another diffusion coverage job if a second pilot is needed again.) This is a +pilot, not a rollout: no dashboard/visualization lives in this repo, and nothing +is gated on coverage. **Naming convention**: `coverage---.xml.gz`, where `` is the model's directory name under @@ -410,7 +412,8 @@ runs a coverage job needs the same. List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's `source_file_dependencies` — both change what the job measures, so without them a change there is filtered out of normal PR builds and only surfaces in a later -nightly. `tests/buildkite/test_upload_pipeline.py` asserts this for the pilots. +nightly. `tests/buildkite/test_upload_pipeline.py` covers the filter behavior with +a synthetic job (it does not pin real merge labels). Editing only the surrounding CI YAML still does not schedule them, so a PR that touches just the wiring needs a full E2E run (or the commands run on a GPU host) to produce artifacts. When checking a new model's diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index ee45a64d3..bd0f50f5c 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -39,8 +39,6 @@ def _render(changed_files: list[str]) -> str: def test_bootstrap_if_injected_by_step_key() -> None: - import yaml - rendered = _render_bootstrap_pipeline( BOOTSTRAP_STEPS_TEMPLATE, decision=resolve_ci_decision([]), @@ -142,12 +140,7 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None: ] -COVERAGE_PILOT_LABELS = ("Diffusion · Bagel Test", "TTS · Qwen3-TTS Base Test") -MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml") - - -def _surviving_labels(changed_files: list[str]) -> set[str]: - doc = yaml.safe_load(MERGE_PIPELINE.read_text(encoding="utf-8")) +def _surviving_labels(doc: dict, changed_files: list[str]) -> set[str]: rendered = _render_test_pipeline(doc, changed_files=changed_files) labels: set[str] = set() @@ -163,6 +156,29 @@ def _surviving_labels(changed_files: list[str]) -> set[str]: return labels +# Synthetic coverage-style job: shared inputs that change what the split measures. +_COVERAGE_SHARED_INPUTS_DOC = { + "steps": [ + { + "label": "Coverage Pilot", + "source_file_dependencies": [ + "tests/e2e/online_serving/test_example.py", + ".buildkite/common/scripts/run_cov_split.sh", + "pyproject.toml", + ], + "commands": [".buildkite/common/scripts/run_cov_split.sh --model-id example"], + }, + { + "label": "Unrelated Model Test", + "source_file_dependencies": [ + "tests/e2e/online_serving/test_other.py", + ], + "commands": ["pytest -sv tests/e2e/online_serving/test_other.py"], + }, + ], +} + + @pytest.mark.parametrize( "changed_file", [ @@ -170,18 +186,17 @@ def _surviving_labels(changed_files: list[str]) -> set[str]: "pyproject.toml", ], ) -def test_coverage_pilots_selected_by_their_shared_inputs(changed_file: str) -> None: - """The split helper and the coverage config both change what these jobs measure. - - Without them in source_file_dependencies the jobs are filtered out of normal - PR builds, so a regression would only surface in a later nightly. - """ - labels = _surviving_labels([changed_file]) - for pilot in COVERAGE_PILOT_LABELS: - assert pilot in labels, f"{pilot} dropped when {changed_file} changed" +def test_coverage_shared_inputs_select_dependent_job(changed_file: str) -> None: + """Jobs that list coverage shared inputs must stay selected when those files change.""" + labels = _surviving_labels(_COVERAGE_SHARED_INPUTS_DOC, [changed_file]) + assert "Coverage Pilot" in labels + assert "Unrelated Model Test" not in labels -def test_coverage_pilots_not_selected_by_unrelated_change() -> None: - labels = _surviving_labels(["vllm_omni/entrypoints/openai/serving_chat.py"]) - for pilot in COVERAGE_PILOT_LABELS: - assert pilot not in labels +def test_coverage_shared_inputs_ignored_for_unrelated_change() -> None: + labels = _surviving_labels( + _COVERAGE_SHARED_INPUTS_DOC, + ["vllm_omni/entrypoints/openai/serving_chat.py"], + ) + assert "Coverage Pilot" not in labels + assert "Unrelated Model Test" not in labels diff --git a/tests/e2e/offline_inference/test_bagel.py b/tests/e2e/offline_inference/test_bagel.py index 9ba7bdac4..580a46742 100644 --- a/tests/e2e/offline_inference/test_bagel.py +++ b/tests/e2e/offline_inference/test_bagel.py @@ -46,7 +46,7 @@ REFERENCE_TEXT_IMG2TEXT = ( _OMNI_RUNNER_PARAM = (MODEL_NAME, STAGE_CONFIG) pytestmark = [ - pytest.mark.advanced_model, + pytest.mark.slow, pytest.mark.diffusion, pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), ] diff --git a/tests/e2e/offline_inference/test_flux2_klein.py b/tests/e2e/offline_inference/test_flux2_klein.py index 9656772c9..ea3943454 100644 --- a/tests/e2e/offline_inference/test_flux2_klein.py +++ b/tests/e2e/offline_inference/test_flux2_klein.py @@ -72,7 +72,7 @@ def _send_inpaint_with_generator( ) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandler): @@ -93,7 +93,7 @@ def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandle ) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_flux2_klein_inpaint_different_seeds_different_output(omni_runner_handler: OmniRunnerHandler): diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index c10694a40..af55c51aa 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -39,8 +39,7 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor: return audio -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: @@ -53,8 +52,7 @@ def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s" -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: @@ -95,8 +93,7 @@ def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s" -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voxcpm2_prefill_decode_mixed_batch_003(omni_runner: OmniRunner) -> None: diff --git a/tests/e2e/offline_inference/test_z_image.py b/tests/e2e/offline_inference/test_z_image.py index 1770ccd85..edf158ffd 100644 --- a/tests/e2e/offline_inference/test_z_image.py +++ b/tests/e2e/offline_inference/test_z_image.py @@ -11,7 +11,7 @@ MODEL = "Tongyi-MAI/Z-Image-Turbo" PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard." -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2}) @pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True) diff --git a/tests/e2e/online_serving/test_bagel.py b/tests/e2e/online_serving/test_bagel.py index b22762acd..dca846108 100644 --- a/tests/e2e/online_serving/test_bagel.py +++ b/tests/e2e/online_serving/test_bagel.py @@ -75,8 +75,7 @@ def _build_img2img_messages(prompt: str, image_b64: str) -> list[dict]: ] -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @@ -98,8 +97,7 @@ def test_bagel_text2img_online(omni_server, openai_client) -> None: openai_client.send_diffusion_request(request_config) -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) diff --git a/tests/e2e/online_serving/test_higgs_audio_v3.py b/tests/e2e/online_serving/test_higgs_audio_v3.py index d26e6bff4..d1d592d21 100644 --- a/tests/e2e/online_serving/test_higgs_audio_v3.py +++ b/tests/e2e/online_serving/test_higgs_audio_v3.py @@ -59,8 +59,7 @@ _REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know w class TestHiggsAudioV3OnlineHappyPath: """Plain-text -> audio happy paths against the live HTTP server.""" - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_wav(self, omni_server, openai_client) -> None: @@ -79,7 +78,7 @@ class TestHiggsAudioV3OnlineHappyPath: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None: @@ -97,8 +96,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_plain_text(self, omni_server, openai_client) -> None: @@ -115,7 +113,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None: @@ -147,7 +145,7 @@ class TestHiggsAudioV3OnlineHappyPath: ) @pytest.mark.skip(reason="issue#4411") - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None: @@ -189,7 +187,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: They do not assert audio quality (out of scope for CI). """ - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None: @@ -205,7 +203,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_style_whispering(self, omni_server, openai_client) -> None: @@ -221,7 +219,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None: @@ -239,7 +237,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_pause_mid_text(self, omni_server, openai_client) -> None: @@ -255,7 +253,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None: @@ -279,8 +277,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens: class TestHiggsAudioV3OnlineVoiceClone: """Voice clone via the two payload shapes the model serves.""" - @pytest.mark.core_model - @pytest.mark.advanced_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None: @@ -297,7 +294,7 @@ class TestHiggsAudioV3OnlineVoiceClone: } ) - @pytest.mark.core_model + @pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_voice_clone_references_alias(self, omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen_image_edit.py b/tests/e2e/online_serving/test_qwen_image_edit.py index 738a2e50c..a02bc1b57 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit.py +++ b/tests/e2e/online_serving/test_qwen_image_edit.py @@ -6,7 +6,7 @@ Online serving tests for the Qwen-Image-Edit family (image-to-image via chat com - ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512. - ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512. -- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input +- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). @@ -52,7 +52,7 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -82,8 +82,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA openai_client.send_diffusion_request(request_config) -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -114,7 +113,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", diff --git a/tests/e2e/online_serving/test_qwen_image_layered.py b/tests/e2e/online_serving/test_qwen_image_layered.py index b29609351..289e35764 100644 --- a/tests/e2e/online_serving/test_qwen_image_layered.py +++ b/tests/e2e/online_serving/test_qwen_image_layered.py @@ -44,8 +44,7 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_voxcpm2_tts.py b/tests/e2e/online_serving/test_voxcpm2_tts.py index 19ddc0401..5597bf367 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts.py @@ -46,8 +46,7 @@ tts_server_params = [ ] -@pytest.mark.advanced_model -@pytest.mark.core_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) @@ -72,7 +71,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.tts @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) diff --git a/tests/e2e/online_serving/test_z_image.py b/tests/e2e/online_serving/test_z_image.py index 2ef78ce0f..b1180f514 100644 --- a/tests/e2e/online_serving/test_z_image.py +++ b/tests/e2e/online_serving/test_z_image.py @@ -32,7 +32,7 @@ def _get_diffusion_feature_cases(): ] -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -60,7 +60,7 @@ def test_basic_001( openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server",