diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5cc52f2d5..0b880ee77 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -31,7 +31,7 @@ steps: key: upload-ready-pipeline if: __UPLOAD_READY_IF__ commands: - - buildkite-agent pipeline upload .buildkite/test-ready.yml + - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/rebase-pipeline.yaml b/.buildkite/rebase-pipeline.yaml index 3ae553fd3..f8f4aa217 100644 --- a/.buildkite/rebase-pipeline.yaml +++ b/.buildkite/rebase-pipeline.yaml @@ -14,7 +14,7 @@ steps: depends_on: image-build key: upload-ready-pipeline commands: - - buildkite-agent pipeline upload .buildkite/test-ready.yml + - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml agents: queue: "cpu_queue_premerge" diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 1dd5caa66..f01a07260 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -44,9 +44,6 @@ steps: depends_on: upload-merge-pipeline steps: - label: "Diffusion · LoRA Test" - source_file_dependencies: - - tests/diffusion/lora/ - - vllm_omni/diffusion/lora/ timeout_in_minutes: 30 commands: - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" @@ -64,9 +61,6 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Distributed Test" - source_file_dependencies: - - tests/distributed/ - - vllm_omni/distributed/ timeout_in_minutes: 60 commands: - | @@ -111,9 +105,6 @@ steps: - group: ":card_index_dividers: Entrypoints Test" depends_on: upload-merge-pipeline - source_file_dependencies: - - tests/entrypoints/ - - vllm_omni/entrypoints/ steps: - label: "Entrypoint Test with H100" timeout_in_minutes: 60 @@ -174,13 +165,6 @@ steps: depends_on: upload-merge-pipeline steps: - label: "TTS · Qwen3-TTS CustomVoice Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - - vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml commands: - | timeout 20m bash -c " @@ -202,12 +186,6 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "TTS · Qwen3-TTS Base Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_base.py - - tests/e2e/offline_inference/test_qwen3_tts_base.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml commands: - | timeout 20m bash -c " @@ -433,11 +411,6 @@ steps: type: DirectoryOrCreate - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_voxcpm2_tts.py - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml timeout_in_minutes: 30 commands: - | @@ -461,12 +434,6 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Omni · Higgs-Audio V2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v2.py - - tests/e2e/offline_inference/test_higgs_audio_v2.py - - vllm_omni/model_executor/models/higgs_audio_v2/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py - - vllm_omni/deploy/higgs_audio_v2.yaml timeout_in_minutes: 30 commands: - | @@ -838,10 +805,6 @@ steps: type: DirectoryOrCreate - label: "Diffusion · Z Image Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_z_image.py - - tests/e2e/online_serving/test_z_image.py - - vllm_omni/diffusion/models/z_image/ timeout_in_minutes: 30 commands: - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' @@ -859,10 +822,6 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion · FLUX.2 Klein Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_flux2_klein.py - - tests/e2e/online_serving/test_flux2_klein.py - - vllm_omni/diffusion/models/flux2_klein/ timeout_in_minutes: 30 commands: - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 1881b4496..699b5e94b 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -215,6 +215,43 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: "Entrypoints Test" + depends_on: upload-ready-pipeline + commands: + - | + timeout 30m bash -c " + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' + " + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - group: ":card_index_dividers: E2E Test" depends_on: upload-ready-pipeline steps: @@ -239,6 +276,11 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Omni · Qwen3-Omni Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen3_omni.py + - vllm_omni/model_executor/models/qwen3_omni/ + - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py + - vllm_omni/deploy/qwen3_omni_moe.yaml commands: - | timeout 20m bash -c " @@ -273,42 +315,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: "Omni · Qwen3-Omni Realtime WebSocket" - commands: - - | - timeout 30m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_streaming_audio_input_pcm_output_async_chunk_smoke -m 'core_model' --run-level 'core_model' - " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: "TTS · VoxCPM2 Test" timeout_in_minutes: 30 commands: @@ -400,6 +406,9 @@ steps: - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion · Qwen Image Edit Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen_image_edit.py + - vllm_omni/diffusion/models/qwen_image/ commands: - | timeout 20m bash -c " @@ -435,6 +444,12 @@ steps: type: DirectoryOrCreate - label: "Diffusion · Bagel Test" + source_file_dependencies: + - tests/e2e/online_serving/test_bagel.py + - vllm_omni/model_executor/models/bagel/ + - vllm_omni/diffusion/models/bagel/ + - vllm_omni/model_executor/stage_input_processors/bagel.py + - vllm_omni/deploy/bagel.yaml commands: - | timeout 40m bash -c " @@ -476,6 +491,9 @@ steps: type: DirectoryOrCreate - label: "Diffusion · Wan22 Test" + source_file_dependencies: + - tests/e2e/online_serving/test_wan22_t2v.py + - vllm_omni/diffusion/models/wan2_2/ commands: - | timeout 40m bash -c " @@ -518,6 +536,9 @@ steps: - label: "Diffusion · Qwen Image Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen_image.py + - vllm_omni/diffusion/models/qwen_image/ commands: - | timeout 40m bash -c " @@ -559,6 +580,9 @@ steps: type: DirectoryOrCreate - label: "Diffusion · Qwen Image Layered Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen_image_layered.py + - vllm_omni/diffusion/models/qwen_image/ commands: - | timeout 40m bash -c " diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index 1ce8002a8..2efa39633 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -268,6 +268,78 @@ Before entering specific testing levels, the project establishes two common spec 1. ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance. 2. ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency. +<<<<<<< ready +## Notes +### Diff-aware Buildkite uploads (`source_file_dependencies`) + +L2 (`.buildkite/test-ready.yml`) and L3 (`.buildkite/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`. + +#### What `source_file_dependencies` is + +- A **uploader-only** YAML key on a Buildkite step or group. **Buildkite does not understand it**; `upload_pipeline.py` always **removes** it from the YAML that is uploaded. +- A list of path **prefixes** (directories or individual files). If **any** changed file in the diff equals a prefix or starts with `prefix/`, the step is kept; otherwise the step (or entire group) is **omitted** from the uploaded pipeline. + +#### When filtering runs + +| Build context | Changed files used for matching | +| --- | --- | +| Pull request | `git diff --name-only origin/...` | +| `main` branch push | `git diff --name-only ^..` | +| Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped | + +Docs-only PRs are handled earlier in bootstrap (`pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines. + +#### Where it is configured + +| Level | Pipeline file | Upload entry | +| --- | --- | --- | +| L2 | `.buildkite/test-ready.yml` | `upload_pipeline.py --upload .buildkite/test-ready.yml` (from `pipeline.yml`) | +| L3 | `.buildkite/test-merge.yml` | `upload_pipeline.py --upload .buildkite/test-merge.yml` (from `pipeline.yml`) | + +Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3). + +#### Current skip policy (L2 / L3) + +To balance CI cost and coverage: + +- **Always run** (no `source_file_dependencies`): baseline groups such as Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3), and **all E2E jobs on L4 docker queues** (`gpu_1_queue`, `gpu_4_queue`). +- **Diff-gated** (`source_file_dependencies` set): **H100 E2E jobs only** (`queue: mithril-h100-pool` under the E2E Test group)—typically the model’s pytest file(s) plus related `vllm_omni/` model, deploy, and stage paths. + +Adding a new expensive H100 E2E step: list the test file(s) and the smallest set of source prefixes that should trigger it. Prefer **per-step** deps on leaf jobs rather than a broad group-level list unless every child shares the same paths. + +#### YAML example + +```yaml + - label: "Diffusion · Qwen Image Test" + source_file_dependencies: + - tests/e2e/online_serving/test_qwen_image.py + - vllm_omni/diffusion/models/qwen_image/ + commands: + - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ... + agents: + queue: "mithril-h100-pool" +``` + +A **group** may also define `source_file_dependencies`; nested steps inherit filtering as a unit—the whole group is dropped if no prefix matches. + +#### Local dry-run + +```bash +# Render filtered YAML to stdout (no upload) +python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-ready.yml + +# Confirm uploader-only keys are stripped +python3 .buildkite/scripts/upload_pipeline.py .buildkite/test-merge.yml | grep source_file_dependencies +# (no output expected) +``` + +On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `skip '…' (no changes under …)` for omitted steps. + +#### Related + +- Implementation: `.buildkite/scripts/upload_pipeline.py` +- L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled. +======= ### Test helper environment variables Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default. @@ -289,6 +361,7 @@ Example (Windows PowerShell): $env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1" pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio ``` +>>>>>>> main ## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification @@ -324,6 +397,7 @@ A clear directory structure is key to managing test cases efficiently. ### 1.4 Execution Method and Example - ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered. +- ***Diff-aware step skipping***: On L2, expensive **H100 E2E** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) paths; L4 docker jobs and non-E2E steps still run. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies). - ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment. - ***Script Example***: @@ -426,7 +500,8 @@ L3 level testing executes after code is merged into the main branch. Its core pu ### 2.4 Execution Method and Example -- ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch. +- ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge). +- ***Diff-aware step skipping***: Same [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) mechanism as L2—only **H100 E2E** steps are diff-gated; L4 docker E2E and other L3 groups always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies). - ***Execution Environment***: ***GPU*** servers. - ***Script Example***: diff --git a/pyproject.toml b/pyproject.toml index 757f038af..1bb8fb2f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,8 @@ dev = [ "voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests "torchdiffeq>=0.2.5", # for covo_audio Test "FlagEmbedding", # for hunyuan_image3 tests - "kernels<0.15.0" # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) + "kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0) + "s3tokenizer>=0.3.0" # for tts test ] demo = [