[BugFix][CI] Restore diff-aware source filtering for post-merge L3 (#7371)

Signed-off-by: wangyu <410167048@qq.com>

.buildkite/common/ci_source_file_dependencies.yml

@@ -11,6 +11,8 @@
 # YAML anchors are nested lists; the loader flattens them.
 
 x-qwen3-tts: &qwen3_tts
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
   - vllm_omni/worker/gpu_model_runner.py
   - vllm_omni/model_executor/models/qwen3_tts/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
@@ -23,6 +25,8 @@ x-qwen3-tts: &qwen3_tts
   - vllm_omni/engine/stage_pool.py
 
 x-minicpmo-4-5: &minicpmo_4_5
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
   - vllm_omni/model_executor/models/minicpmo_4_5/
   - vllm_omni/model_executor/models/step_audio2/
   - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
@@ -43,7 +47,6 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
   - vllm_omni/core/sched/
   - vllm_omni/outputs/multimodal_accumulation.py
   - vllm_omni/engine/stage_init_utils.py
-  - vllm_omni/engine/async_omni_engine.py
   - vllm_omni/engine/orchestrator.py
   - vllm_omni/entrypoints/cli/serve.py
   - vllm_omni/entrypoints/openai/api_server.py
@@ -51,11 +54,15 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
   - vllm_omni/config/stage_config.py
 
 x-qwen2-5-omni: &qwen2_5_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
   - vllm_omni/model_executor/models/qwen2_5_omni/
   - vllm_omni/model_executor/stage_input_processors/qwen2_5_omni.py
   - vllm_omni/deploy/qwen2_5_omni.yaml
 
 x-qwen3-omni: &qwen3_omni
+  - vllm_omni/engine/async_omni_engine.py
+  - vllm_omni/entrypoints/omni_base.py
   - vllm_omni/entrypoints/openai/serving_chat.py
   - vllm_omni/model_executor/models/qwen3_omni/
   - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
@@ -116,11 +123,9 @@ x-lingbot-world: &lingbot_world
 
 
 source_file_dependencies:
-  # Shared uploader/registry paths. Not a job key — do not attach this to a step.
-  # Used only when no job-key prefix matches: then a hit keeps every job.
+  # Hardware presets and shared test helpers. Not a job key — do not attach this
+  # to a step. Used only when no job-key prefix matches: then a hit keeps every job.
   source_filter_fallback:
-    - .buildkite/common/scripts/upload_pipeline.py
-    - .buildkite/common/ci_source_file_dependencies.yml
     - .buildkite/common/ci_mirror_hardwares.yml
     - tests/helpers/
 
@@ -280,6 +285,7 @@ source_file_dependencies:
     - *wan22
     - tests/e2e/offline_inference/test_wan22_t2v.py
     - tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py
+    - tests/e2e/offline_inference/test_device_postprocess_equivalence.py
     - tests/e2e/online_serving/test_wan22_t2v.py
     - tests/e2e/online_serving/test_wan22_expansion.py
 

.buildkite/common/scripts/upload_pipeline.py

@@ -13,12 +13,14 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
 Test pipeline mode (e.g. test-merge.yml, test-nightly.yml, test-weekly.yml):
   - Drop steps whose ``source_file_dependencies`` do not match changed files
     (string/list keys from ci_source_file_dependencies.yml, or inline path
-    prefixes). Filtering applies on **PR label** uploads only; ``main`` + env
-    schedule (NIGHTLY/WEEKLY/merge push) keeps every job and still strips the field.
-    If no job-key prefix matches, a change to the pipeline YAML being uploaded
-    or to a path under the ``source_filter_fallback`` registry key keeps every
-    job so command, env, and hardware edits can be validated before merge.
-    When any job-key prefix already matches, normal filtering wins.
+    prefixes). Filtering applies on **PR label** uploads and on post-merge
+    ``main`` L3 uploads. Scheduled ``main`` uploads that pass ``--all``
+    (NIGHTLY/WEEKLY) or ``--e2e`` keep every selected job and still strip
+    the field. If no job-key prefix matches, a change to the pipeline YAML
+    being uploaded or to a path under the ``source_filter_fallback`` registry
+    key keeps every job so command, env, and hardware edits can be validated
+    before merge. When any job-key prefix already matches, normal filtering
+    wins.
   - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
   - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
@@ -780,17 +782,14 @@ def _changed_files_for_source_filter(
 ) -> list[str] | None:
     """Return the diff to filter against, or None to keep every step.
 
-    ``source_file_dependencies`` is label-only: scheduled ``main`` uploads
-    (NIGHTLY/WEEKLY/post-merge) run the full pipeline. ``--all`` / ``--e2e``
-    also disable filtering. Pipeline YAML / ``source_filter_fallback`` matches
-    are applied later in ``_render_test_pipeline`` only when no job-key prefix
-    already matched.
+    ``--all`` / ``--e2e`` disable filtering (scheduled NIGHTLY/WEEKLY full
+    uploads and weekly E2E sweeps). Post-merge ``main`` L3 and PR-label
+    uploads filter against the commit/PR diff. Pipeline YAML /
+    ``source_filter_fallback`` matches are applied later in
+    ``_render_test_pipeline`` only when no job-key prefix already matched.
     """
     if force_all or e2e_only:
         return None
-    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
-        _log("main branch: keep all jobs (source_file_dependencies is label-only)")
-        return None
     return ctx.changed_files
 
 

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -22,19 +22,16 @@ steps:
     agents:
       queue: "cpu_queue_premerge"
 
-  # L3 Test — PR merge-test label (diff-aware) or main (full pipeline / WEEKLY --e2e)
+  # L3 Test — PR merge-test label or post-merge on main (both diff-aware);
+  # main+WEEKLY=1 uploads the E2E group only (--e2e).
   - label: "Upload Merge Pipeline"
     depends_on: image-build
     key: upload-merge-pipeline
     commands:
       - |
         EXTRA=""
-        if [ "${BUILDKITE_BRANCH}" = "main" ]; then
-          if [ "${WEEKLY:-}" = "1" ]; then
-            EXTRA="--e2e"
-          else
-            EXTRA="--all"
-          fi
+        if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
         fi
         python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
     agents:

.buildkite/cuda/test-merge.yml

@@ -1,5 +1,6 @@
 # CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
-# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main
+# (both diff-aware via source_file_dependencies).
 
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn

.buildkite/cuda/test-nightly.yml

@@ -390,6 +390,7 @@ steps:
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test"
+        source_file_dependencies: diffusion_wan22_function
         timeout_in_minutes: 90
         commands:
           - |

docs/contributing/ci/ci_settings.md

@@ -313,7 +313,7 @@ Unit coverage: `tests/buildkite/test_skip_ci.py`.
 
 CUDA **L2** (`.buildkite/cuda/test-ready.yml`), **L3** (`.buildkite/cuda/test-merge.yml`), **L4** (`.buildkite/cuda/test-nightly.yml`), **L5** (`.buildkite/cuda/test-weekly.yml`), and NPU **L4** (`.buildkite/npu/test-npu-nightly.yml`). Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml` (or `npu/bootstrap-upload-steps.yml`).
 
-**When filtering applies:** only **PR label** uploads (`ready`, `merge-test`, `nightly-test`, `weekly-test`). On **`main` + env** (`NIGHTLY=1`, post-merge L3, `WEEKLY=1` / `NON_CRITICAL=1`, `WEEKLY=1 --e2e`) the uploader keeps every job and still strips the key. If **no** job-key prefix matches the diff, a change to the pipeline YAML being uploaded, or any path under the `source_filter_fallback` key in `ci_source_file_dependencies.yml`, keeps every job in that upload. `source_filter_fallback` is not a job key — do not attach it to a step. When any job-key prefix already matches, normal filtering wins (the fallback does not expand the selection). Without this fallback, command / env / hardware-only edits would match nothing (NPU drops every step; CUDA nightly keeps only the email aggregator). A different pipeline YAML does not trigger the fallback for this upload.
+**When filtering applies:** **PR label** uploads (`ready`, `merge-test`, `nightly-test`, `weekly-test`) and **post-merge `main` L3** (`test-merge.yml`). Scheduled **`main` + env** uploads that pass `--all` (`NIGHTLY=1`, `WEEKLY=1` / `NON_CRITICAL=1`) or `--e2e` (`WEEKLY=1` on L2/L3) keep every selected job and still strip the key. If **no** job-key prefix matches the diff, a change to the pipeline YAML being uploaded, or any path under the `source_filter_fallback` key in `ci_source_file_dependencies.yml`, keeps every job in that upload. `source_filter_fallback` is not a job key — do not attach it to a step. When any job-key prefix already matches, normal filtering wins (the fallback does not expand the selection).
 
 **Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents:
 

tests/buildkite/test_upload_pipeline.py

@@ -692,19 +692,24 @@ def test_source_filter_strips_deps_and_expands_hardware(monkeypatch: pytest.Monk
     assert z_image["agents"]["queue"] == "l4-k8s"
 
 
-def test_source_filter_disabled_on_main_branch(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_source_filter_respects_force_all_and_uses_diff_on_main(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     class _Ctx:
         changed_files = ["vllm_omni/unrelated.py"]
 
+    # Post-merge main L3 must still filter by the commit diff.
     monkeypatch.setenv("BUILDKITE_BRANCH", "main")
-    assert _changed_files_for_source_filter(_Ctx(), force_all=False, e2e_only=False) is None
+    assert _changed_files_for_source_filter(_Ctx(), force_all=False, e2e_only=False) == [
+        "vllm_omni/unrelated.py",
+    ]
 
     monkeypatch.setenv("BUILDKITE_BRANCH", "feat/source-filter")
     assert _changed_files_for_source_filter(_Ctx(), force_all=False, e2e_only=False) == [
         "vllm_omni/unrelated.py",
     ]
-    monkeypatch.setenv("BUILDKITE_BRANCH", "main")
     assert _changed_files_for_source_filter(_Ctx(), force_all=True, e2e_only=False) is None
+    assert _changed_files_for_source_filter(_Ctx(), force_all=False, e2e_only=True) is None
 
 
 def test_source_filter_fallback_is_fallback_when_no_job_key_matches(
@@ -715,7 +720,6 @@ def test_source_filter_fallback_is_fallback_when_no_job_key_matches(
     monkeypatch.setenv("BUILDKITE_BRANCH", "feat/nightly-yaml")
     pipeline_yaml = Path(".buildkite/npu/test-npu-nightly.yml")
     shared_paths = _load_source_file_dependencies()["source_filter_fallback"]
-    assert ".buildkite/common/scripts/upload_pipeline.py" in shared_paths
 
     # Synthetic steps only — do not pin live Buildkite job labels.
     doc = {
@@ -742,7 +746,7 @@ def test_source_filter_fallback_is_fallback_when_no_job_key_matches(
         )
         return {step["key"] for step in _iter_steps(rendered) if isinstance(step.get("key"), str)}
 
-    # Only pipeline YAML / shared uploader → no listed prefix match → keep every step.
+    # Only pipeline YAML / fallback paths → no listed prefix match → keep every step.
     for changed in [pipeline_yaml.as_posix(), *shared_paths]:
         assert surviving_keys([changed]) == {"ungated", "gated_a", "gated_b"}, changed
 
@@ -750,9 +754,7 @@ def test_source_filter_fallback_is_fallback_when_no_job_key_matches(
     assert surviving_keys([".buildkite/cuda/test-nightly.yml"]) == {"ungated"}
 
     # A matching source prefix wins over fallback files in the same diff.
-    assert surviving_keys(
-        [
-            "pkg/model_a/transformer.py",
-            ".buildkite/common/scripts/upload_pipeline.py",
-        ],
-    ) == {"ungated", "gated_a"}
+    assert surviving_keys(["pkg/model_a/transformer.py", *shared_paths[:1]]) == {
+        "ungated",
+        "gated_a",
+    }