diff --git a/.buildkite/amd/test-amd-merge.yml b/.buildkite/amd/test-amd-merge.yml index fbe3b0c0f..2bc5a4871 100644 --- a/.buildkite/amd/test-amd-merge.yml +++ b/.buildkite/amd/test-amd-merge.yml @@ -17,7 +17,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - label: "Simple · Diffusion Test · Shard %N/%t" agent_pool: mi300_1 @@ -29,7 +29,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-diffusion-$$BUILDKITE_PARALLEL_JOB.xml" + - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - label: "Simple · Engine&Entrypoints Test" agent_pool: mi300_1 @@ -38,7 +38,7 @@ steps: grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 - - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" - label: "Simple · Other Test" agent_pool: mi300_1 @@ -49,7 +49,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py" - group: ":card_index_dividers: Diffusion Test" steps: diff --git a/.buildkite/amd/test-amd-ready.yml b/.buildkite/amd/test-amd-ready.yml index b8fca2f88..68ceed718 100644 --- a/.buildkite/amd/test-amd-ready.yml +++ b/.buildkite/amd/test-amd-ready.yml @@ -17,7 +17,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" - label: "Simple · Diffusion Test" agent_pool: mi300_1 @@ -28,7 +28,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py" - label: "Simple · Engine&Entrypoints Test" agent_pool: mi300_1 @@ -37,7 +37,7 @@ steps: grade: Blocking commands: - export VLLM_ROCM_USE_AITER=0 - - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" - label: "Simple · Other Test" agent_pool: mi300_1 @@ -48,7 +48,7 @@ steps: - export VLLM_ROCM_USE_AITER=0 # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM # that is not supported on CPU - - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --ignore=tests/diffusion/cache/test_teacache_extractors.py" - label: "Custom Pipeline Test" agent_pool: mi300_1 diff --git a/.buildkite/common/scripts/run_cov_split.sh b/.buildkite/common/scripts/run_cov_split.sh index fe068519c..0ed61ec17 100755 --- a/.buildkite/common/scripts/run_cov_split.sh +++ b/.buildkite/common/scripts/run_cov_split.sh @@ -16,6 +16,11 @@ # At least one of --offline/--online is required, so a job with only one mode runs # just that half. Total runtime is bounded by the step's timeout_in_minutes. # +# Coverage + artifact upload run only on scheduled weekly E2E sweeps +# (WEEKLY=1 and BUILDKITE_BRANCH=main — same gate as bootstrap --e2e). +# All other invocations (PR merge-test, regular main merge, local) run the +# same combined pytest as the job used before coverage was added. +# # Exits non-zero if any half or the upload failed. set -uo pipefail @@ -44,6 +49,17 @@ if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then exit 2 fi +_should_run_coverage() { + [[ "${WEEKLY:-}" == "1" && "${BUILDKITE_BRANCH:-}" == "main" ]] +} + +if ! _should_run_coverage; then + PATHS=("${OFFLINE[@]}" "${ONLINE[@]}") + echo "--- pytest (no coverage): ${MODEL_ID}" + pytest -s -v "${PATHS[@]}" "${PYTEST_ARGS[@]}" + exit $? +fi + # Two steps can test the same model, and artifacts are scoped to the build (a # nightly one carries the ready, merge and nightly tiers at once), so unsuffixed # reports would share a path -- a glob download then races to rename onto one file. diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index b27f78c3a..a9c0b0561 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Render and optionally upload Buildkite pipeline YAML with diff-aware logic. Bootstrap mode (``bootstrap-upload-steps.yml``): @@ -58,11 +59,12 @@ BOOTSTRAP_UPLOAD_IF_KEYS = { E2E_GROUP_MARKER = "E2E Test" CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" -CUDA_NIGHTLY_ONLY = ( - '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")' -) -NPU_NIGHTLY_ONLY = ( - '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' +# Bootstrap Buildkite ``if`` expressions. +# ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN). +# ``*_UPLOAD_IF``: full gate for uploading that child pipeline. +NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"' +NIGHTLY_LABEL_IF = ( + f"({NIGHTLY_MAIN_IF}) || " '(build.branch != "main" && (' 'build.pull_request.labels includes "nightly-test" || ' 'build.pull_request.labels includes "omni-test" || ' @@ -71,6 +73,19 @@ NPU_NIGHTLY_ONLY = ( 'build.pull_request.labels includes "diffusion-x2v-test"' "))" ) +WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"' +WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")' +WEEKLY_LABEL_IF = f'({WEEKLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "weekly-test")' +READY_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "ready"' +MERGE_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "merge-test"' +MERGE_MAIN_IF = ( + 'build.branch == "main" && build.env("NIGHTLY") != "1" && ' + 'build.env("WEEKLY") != "1" && build.env("NON_CRITICAL") != "1"' +) +READY_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || ({READY_LABEL_IF})" +MERGE_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || (({MERGE_MAIN_IF}) || ({MERGE_LABEL_IF}))" +BOOTSTRAP_DISABLED_IF = "false" +BOOTSTRAP_ENABLED_IF = "true" # --- Logging --- @@ -102,74 +117,47 @@ def _format_bootstrap_if(expr: str) -> str: def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: - disabled = "false" - nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"' - nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY - if platform == "npu": - ready_pr = ( - 'build.branch != "main" && (' - 'build.pull_request.labels includes "npu-test" || ' - 'build.pull_request.labels includes "ready"' - ")" - ) - nightly_label_if = nightly_only - weekly_label_if = disabled - merge_base = disabled + ready_upload = READY_LABEL_IF + merge_upload = BOOTSTRAP_DISABLED_IF + weekly_label_if = BOOTSTRAP_DISABLED_IF else: - ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' - merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"' - merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"' - nightly_label_if = ( - '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' - '(build.branch != "main" && (' - 'build.pull_request.labels includes "nightly-test" || ' - 'build.pull_request.labels includes "omni-test" || ' - 'build.pull_request.labels includes "tts-test" || ' - 'build.pull_request.labels includes "diffusion-x2iat-test" || ' - 'build.pull_request.labels includes "diffusion-x2v-test"' - "))" - ) - weekly_label_if = ( - '(build.branch == "main" && build.env("WEEKLY") == "1") || ' - '(build.branch != "main" && build.pull_request.labels includes "weekly-test")' - ) - merge_base = f"({nightly_main}) || (({merge_main}) || ({merge_pr}))" - - ready_base = f"({nightly_main}) || ({ready_pr})" - weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"' + ready_upload = READY_UPLOAD_IF + merge_upload = MERGE_UPLOAD_IF + weekly_label_if = WEEKLY_LABEL_IF if decision.skip_all: # Docs / skip-mark only: no PR-label escape hatch. Main scheduled - # NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5. - image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main - ready_expr = nightly_main - merge_expr = nightly_main if platform == "cuda" else disabled - nightly_expr = nightly_main - weekly_expr = weekly_main if platform == "cuda" else disabled + # NIGHTLY=1 still runs L4; WEEKLY=1 / NON_CRITICAL=1 still run L5. + # main+WEEKLY=1 also uploads L2/L3 (those steps then pass --e2e). + image_expr = f"({NIGHTLY_MAIN_IF}) || ({WEEKLY_MAIN_IF})" if platform == "cuda" else NIGHTLY_MAIN_IF + ready_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF + merge_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF + nightly_expr = NIGHTLY_MAIN_IF + weekly_expr = WEEKLY_MAIN_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF elif decision.skip_l2_l3: l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2") l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3") - ready_expr = ready_base if l2_enabled else disabled - merge_expr = merge_base if l3_enabled else disabled - nightly_expr = nightly_label_if - weekly_expr = weekly_label_if if platform == "cuda" else disabled + ready_expr = ready_upload if l2_enabled else BOOTSTRAP_DISABLED_IF + merge_expr = merge_upload if l3_enabled else BOOTSTRAP_DISABLED_IF + nightly_expr = NIGHTLY_LABEL_IF + weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF - image_parts = [f"({nightly_label_if})"] + image_parts = [f"({NIGHTLY_LABEL_IF})"] if platform == "cuda": image_parts.append(f"({weekly_label_if})") if l2_enabled: - image_parts.insert(0, f"({ready_base})") + image_parts.insert(0, f"({ready_upload})") if l3_enabled: - image_parts.insert(1 if l2_enabled else 0, f"({merge_base})") + image_parts.insert(1 if l2_enabled else 0, f"({merge_upload})") image_expr = " || ".join(image_parts) else: - image_expr = "true" - ready_expr = ready_base - merge_expr = merge_base if platform == "cuda" else disabled - nightly_expr = nightly_label_if - weekly_expr = weekly_label_if if platform == "cuda" else disabled + image_expr = BOOTSTRAP_ENABLED_IF + ready_expr = ready_upload + merge_expr = merge_upload if platform == "cuda" else BOOTSTRAP_DISABLED_IF + nightly_expr = NIGHTLY_LABEL_IF + weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF return { "image": _format_bootstrap_if(image_expr), diff --git a/.buildkite/cuda/bootstrap-upload-steps.yml b/.buildkite/cuda/bootstrap-upload-steps.yml index f2467bf42..ab4254feb 100644 --- a/.buildkite/cuda/bootstrap-upload-steps.yml +++ b/.buildkite/cuda/bootstrap-upload-steps.yml @@ -8,28 +8,28 @@ steps: agents: queue: "cpu_queue_premerge" - # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + # L2 Test — PR ready label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group) - label: "Upload Ready Pipeline" depends_on: image-build key: upload-ready-pipeline commands: - | EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" - # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + # L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group) - label: "Upload Merge Pipeline" depends_on: image-build key: upload-merge-pipeline commands: - | EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then EXTRA="--e2e" fi python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml @@ -45,7 +45,7 @@ steps: agents: queue: "cpu_queue_premerge" - # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label + # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label - label: "Upload Weekly Pipeline" depends_on: image-build key: upload-weekly-pipeline diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 6a402254b..2d3be8bc4 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -9,23 +9,27 @@ steps: depends_on: upload-merge-pipeline steps: - label: "Simple · Model Executor Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Diffusion Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/diffusion -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Engine&Entrypoints Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Other Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine" mirror_hardwares: l4_1 - group: ":card_index_dividers: Diffusion Test" diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 986047bf6..9c311376b 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -89,32 +89,6 @@ steps: exit $$EXIT mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Perf Test · vLLM Text" - key: nightly-omni-performance-vllm-text - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_2 - - - label: ":full_moon: Omni · Perf Test · Multi-Replica" - key: nightly-omni-performance-multi-replicas - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_3 - - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test" key: nightly-omni-performance-minicpmo-4-5 timeout_in_minutes: 180 @@ -412,8 +386,6 @@ steps: depends_on: - nightly-omni-performance-no-async-chunk - nightly-omni-performance-async-chunk - - nightly-omni-performance-vllm-text - - nightly-omni-performance-multi-replicas - nightly-tts-performance - nightly-diffusion-x2iat-performance-qwen-image - nightly-diffusion-x2v-performance @@ -425,8 +397,6 @@ steps: - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - python tools/nightly/generate_nightly_perf_excel.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index ec3d7485a..ff55409b3 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -9,23 +9,27 @@ steps: depends_on: upload-ready-pipeline steps: - label: "Simple · Model Executor Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/model_executor -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Diffusion Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/diffusion -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Engine&Entrypoints Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" mirror_hardwares: l4_1 - label: "Simple · Other Test" + timeout_in_minutes: 20 commands: - - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine" mirror_hardwares: l4_1 - group: ":card_index_dividers: Diffusion Test" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index ed20e78dc..0b4ca2740 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -5,51 +5,91 @@ env: HF_HUB_ETAG_TIMEOUT: 60 steps: - - - label: "Reliability Test - qwen3-omni" - timeout_in_minutes: 180 + - label: "Simple · CPU Coverage Test" + key: weekly-simple-cpu-coverage depends_on: upload-weekly-pipeline - if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" + if: build.env("WEEKLY") == "1" + timeout_in_minutes: 60 commands: - - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" - mirror_hardwares: h100_2 + - | + set +e + REPORT="coverage-core-model-cpu-$${BUILDKITE_STEP_ID:-local}.xml" + pytest -sv tests/ -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:$${REPORT} + EXIT=$$? + gzip -9 -f "$${REPORT}" + buildkite-agent artifact upload "$${REPORT}.gz" + exit $$EXIT + mirror_hardwares: l4_1 - - label: "Reliability Test - wan22" - timeout_in_minutes: 180 - depends_on: upload-weekly-pipeline - if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" - commands: - - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" - mirror_hardwares: h100_2 - - - label: "Reliability Test - hunyuan-image" - timeout_in_minutes: 180 - depends_on: upload-weekly-pipeline - if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" - commands: - - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" - mirror_hardwares: h100_4 - - - group: "Reliability Test - Invalid parameters Test" + - group: "Reliability Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" steps: - - label: "Invalid parameters Test · H100" + - label: "Reliability Test · qwen3-omni" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" + mirror_hardwares: h100_2 + + - label: "Reliability Test · wan22" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" + mirror_hardwares: h100_2 + + - label: "Reliability Test · hunyuan-image" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" + mirror_hardwares: h100_4 + + - label: "Reliability Test · Invalid parameters · H100" timeout_in_minutes: 120 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100" mirror_hardwares: h100_2 - - label: "Invalid parameters Test · L4" + - label: "Reliability Test · Invalid parameters · L4" timeout_in_minutes: 120 commands: - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4" mirror_hardwares: l4_1 - - group: ":card_index_dividers: E2E Tests" + - group: "Perf Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" + steps: + + - label: ":full_moon: Omni · Perf Test · vLLM Text" + key: weekly-omni-performance-vllm-text + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + mirror_hardwares: h100_2 + + - label: ":full_moon: Omni · Perf Test · Multi-Replica" + key: weekly-omni-performance-multi-replicas + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + mirror_hardwares: h100_3 + + - group: ":card_index_dividers: E2E Tests" + depends_on: upload-weekly-pipeline + if: build.env("NON_CRITICAL") == "1" steps: - label: "Omni · L4" @@ -75,7 +115,7 @@ steps: mirror_hardwares: h100_2 - label: "TTS · L4" - timeout_in_minutes: 90 + timeout_in_minutes: 120 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" @@ -126,7 +166,7 @@ steps: mirror_hardwares: l4_4 - label: "Diffusion · H100 · Single-GPU" - timeout_in_minutes: 120 + timeout_in_minutes: 150 commands: - >- pytest -sv diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 43afd3e5c..1db082be5 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -334,7 +334,7 @@ These PR shapes all trigger skip-ci: Skip-ci does **not** apply when the diff also touches product code (for example `vllm_omni/`), or when test files change assertions, imports, fixtures, or other non-skip logic. If the diff cannot be resolved (non-PR branches outside `main`), CI runs as usual. !!! note - Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. Bootstrap child steps live in `bootstrap-upload-steps.yml`; the hook entry step runs `upload_pipeline.py --upload /bootstrap-upload-steps.yml`, which injects `if` by step `key` from skip-ci before upload. See [CI Settings — Diff-aware CI](./ci/ci_settings.md#diff-aware-ci) and [Test System Overview](./ci/test_system_overview.md). + Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. Weekly (L5) upload can still run with `weekly-test`, `WEEKLY=1`, or `NON_CRITICAL=1`. Bootstrap child steps live in `bootstrap-upload-steps.yml`; the hook entry step runs `upload_pipeline.py --upload /bootstrap-upload-steps.yml`, which injects `if` by step `key` from skip-ci before upload. See [CI Settings — Diff-aware CI](./ci/ci_settings.md#diff-aware-ci) and [Test System Overview](./ci/test_system_overview.md). ### Code Quality diff --git a/docs/contributing/ci/ci_settings.md b/docs/contributing/ci/ci_settings.md index 035e8c650..246aa4dc9 100644 --- a/docs/contributing/ci/ci_settings.md +++ b/docs/contributing/ci/ci_settings.md @@ -8,7 +8,7 @@ For CI levels (L1–L5) and triggers, see [Test System Overview](./test_system_o Canonical layout (prefer these paths for new changes): -``` +```text .buildkite/ ├── common/ # Shared across platforms │ ├── scripts/ @@ -48,7 +48,7 @@ Canonical layout (prefer these paths for new changes): └── scripts/ ``` -**Placement rules** +### Placement rules | Rule | Detail | | ---- | ------ | @@ -81,7 +81,14 @@ There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` pr | L2 | `cuda/test-ready.yml` | `ready` label | | L3 | `cuda/test-merge.yml` | `merge-test` label / main merge | | L4 | `cuda/test-nightly.yml` | `nightly-test` label or `NIGHTLY=1` | - | L5 | `cuda/test-weekly.yml` | `weekly-test` label or `WEEKLY=1` | + | L5 | `cuda/test-weekly.yml` | `weekly-test` label, `WEEKLY=1`, or `NON_CRITICAL=1` (see step `if`s below) | + + On scheduled `main` builds, `test-weekly.yml` is uploaded when **`WEEKLY=1` or `NON_CRITICAL=1`**. Inside that file: + + - **Reliability** / **Perf Test** / **Simple · CPU Coverage Test** → `WEEKLY=1` (or PR label `weekly-test` for Reliability/Perf). Pipeline upload for scheduled env vars still requires `main`. + - **E2E Tests** group (slow Omni/TTS/Diffusion sweeps) → `NON_CRITICAL=1` + + **`--e2e`:** when uploading ready/merge on `main` with **`WEEKLY=1`**, bootstrap passes `--e2e` so only the **E2E Test** group is kept (same flag `run_cov_split.sh` uses to enable per-model coverage). **Upload:** `upload_pipeline.py --upload` expands uploader-only keys before Buildkite upload. @@ -112,6 +119,8 @@ There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` pr **Test YAML:** `npu/test-npu-ready.yml` (L2), `test-npu-nightly.yml` (L4). + **L2 trigger:** PR label `ready` (CUDA/AMD L2). PR labels `ready` + `npu-test` trigger NPU CI only. + **Upload:** same as CUDA—`upload_pipeline.py --upload`. **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. Presets: `a2b3_npu_1`, `a2b3_npu_4`, `a3_npu_2` in `common/ci_mirror_hardwares.yml`. @@ -204,15 +213,15 @@ Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only red | Diff shape | Path | Default L2/L3 | | --- | --- | --- | | Product code / non-whitelisted paths | normal CI | all on | -| Docs and/or qualifying skip-mark only | `skip_all` | all off (scheduled `main` NIGHTLY/WEEKLY exceptions below) | +| Docs and/or qualifying skip-mark only | `skip_all` | all off (scheduled `main` NIGHTLY/WEEKLY/`NON_CRITICAL` exceptions below) | | Whitelisted CI YAML only | yaml-gated (`skip_l2_l3`) | per platform/level matrix | | Docs/skip-mark **+** whitelisted CI YAML | yaml-gated (`skip_l2_l3`) | same as CI-YAML-only (does **not** widen to normal CI) | | Non-qualifying skip-mark and no CI YAML | normal CI | all on | | Diff unavailable | normal CI | all on | -**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `npu-test`, `weekly-test`, …) do **not** revive jobs. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4 plus L2/L3 (with `--e2e`); `WEEKLY=1` still builds the image and uploads L5 (CUDA). +**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `weekly-test`, `ready`, …) do **not** revive jobs under `skip_all`. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4; `WEEKLY=1` or `NON_CRITICAL=1` still builds the image and uploads L5 (CUDA). `WEEKLY=1` on `main` also uploads L2/L3 with `--e2e`. -**Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated. +**Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` / `NON_CRITICAL` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated. ##### Whitelisted CI YAML → platform @@ -285,12 +294,12 @@ CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test | `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. | | `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. | -**Policy** +### Policy - **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3). - **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches. -**YAML examples** +### YAML examples ```yaml - label: "Diffusion · Qwen Image Test" @@ -312,7 +321,7 @@ CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test mirror_hardwares: l4_4 ``` -**Local dry-run** +### Local dry-run ```bash python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml @@ -324,18 +333,19 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for ### Labels and grouping -- Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main. +- Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` weekly runs on main (`WEEKLY=1`). - Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps. ### Per-model coverage Pilot (v1): one Merge-tier (L3) job in `.buildkite/cuda/test-merge.yml` uploads per-model, per-entry-mode coverage as Buildkite artifacts — "TTS · Qwen3-TTS Base -Test". It runs on a single GPU so the pilot is cheap to reproduce. (A second -diffusion pilot, "Diffusion · Bagel Test", was retired from merge; reintroduce -another diffusion coverage job if a second pilot is needed again.) This is a -pilot, not a rollout: no dashboard/visualization lives in this repo, and nothing -is gated on coverage. +Test". Coverage + artifact upload run only when bootstrap uploaded that job with +`--e2e` (`WEEKLY=1` and `BUILDKITE_BRANCH=main`, via `run_cov_split.sh`); otherwise +the same job runs plain pytest without `--cov`. It runs on a single GPU so the +pilot is cheap to reproduce. + +**L1 package coverage (CPU):** ready/merge Simple Test jobs run without `--cov`. Scheduled weekly on `main` with `WEEKLY=1` runs **Simple · CPU Coverage Test** in `.buildkite/cuda/test-weekly.yml` (`pytest -m 'core_model and cpu'` + Cobertura XML artifact). **Naming convention**: `coverage---.xml.gz`, where `` is the model's directory name under @@ -354,8 +364,8 @@ imported. A run covering zero lines produces the same 7 MB. gzip takes it to roughly 410 KB and changes nothing about the data. `` is what keeps two steps that cover the same model apart. Artifacts are -scoped to the build, and a nightly build carries the ready, merge and nightly tiers -at once, so several same-model steps land in one namespace — `qwen3_tts` already has +scoped to the build, and a weekly build can carry ready/merge (with `--e2e`) and +weekly tiers at once, so several same-model steps land in one namespace — `qwen3_tts` already has both a Base and a CustomVoice job, and `minicpmo_4_5` a base and a duplex one. Two uploads on one path do not fail loudly: an exact-name download can report the path as ambiguous, and a glob download fetches every match concurrently and renames each @@ -425,7 +435,7 @@ just confirming both files exist. | Check | Command / location | | ----- | ------------------ | | CUDA render | `python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-.yml` | -| No leaked uploader keys | `… \| grep -E 'mirror_hardwares|source_file_dependencies'` → empty | +| No leaked uploader keys | Pipe the render into `grep -E 'mirror_hardwares\|source_file_dependencies'` — expect empty | | Skip-ci / upload unit tests | `pytest tests/buildkite/` | | Local job replay (CUDA L2+) | `tools/run_ready_jobs.sh`, `tools/run_merge_jobs.sh`, `tools/nightly/run_nightly_jobs.sh` (read YAML from `cuda/`) | diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 25d72d9be..ec1e3d01b 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -1,14 +1,15 @@ When you want to add L4-level ***performance test*** cases, add entries to JSON files under `tests/dfx/perf/tests/` and run them via `tests/dfx/perf/scripts/run_benchmark.py` (omni / TTS) or `run_diffusion_benchmark.py` (diffusion). -**Config file layout (in-tree examples)** +#### Config file layout (in-tree examples) | Model type | Runner | Example JSON files | | ---------- | ------ | ------------------ | -| Omni | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` | +| Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` in `mark`) | +| Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) | | TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | | Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … | -**How runners pick cases** +#### How runners pick cases Without **`--test-config-file`**, each runner scans all `*.json` under `tests/dfx/perf/tests/` but only keeps its own model type: @@ -17,9 +18,9 @@ Without **`--test-config-file`**, each runner scans all `*.json` under `tests/df Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-omni"`) or `"diffusion"` in the `mark` array. Omni / TTS JSON has neither. -**Running perf cases** +#### Running perf cases -Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"`). +Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"` or `-m "slow and omni and H100"` for weekly Omni configs). ```JSON { @@ -55,20 +56,22 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul } ``` -**Parameter Explanation** +#### Parameter Explanation -*Overview* +##### Overview -| Field | Required | Description | -| ---------------- | -------- | --------------------------------------------------------------- | -| test_name | Yes | Unique identifier for the test case | -| mark | No | Pytest marks for this case (see **`mark` field** below). Omit only for configs not meant to be filtered by `-m`. | -| server_params | Yes | Server-side configuration parameters | -| benchmark_params | Yes | Benchmark running parameters (supports multiple configurations) | -| server_type | Diffusion only | When set (e.g. `"vllm-omni"`), routes the case to `run_diffusion_benchmark.py`. | -| benchmark_endpoint | Diffusion only | API path for the benchmark client (e.g. `/v1/videos`, `/v1/images/generations`). | +| Field | Required | Description | +| ------------------ | -------------- | -------------------------------------------- | +| test_name | Yes | Unique identifier for the test case | +| mark | No | Pytest marks; see **`mark` field** below | +| server_params | Yes | Server-side configuration parameters | +| benchmark_params | Yes | Benchmark running parameters | +| server_type | Diffusion only | Routes case to run_diffusion_benchmark.py | +| benchmark_endpoint | Diffusion only | Benchmark API path | -**`mark` field** +Omit `mark` only for configs not meant to be filtered by `-m`. `server_type` is typically `"vllm-omni"`. `benchmark_endpoint` examples: `/v1/videos`, `/v1/images/generations`. + +#### `mark` field Optional top-level field on each perf JSON **case object** (one per `test_name`). `run_benchmark.py` and `run_diffusion_benchmark.py` read it via `tests.dfx.conftest.resolve_pytest_marks` and attach the marks to the corresponding `pytest.param`, so you can filter locally with `-m`. @@ -119,11 +122,11 @@ Multi-GPU diffusion (example: Cosmos3 with `cfg-parallel-size=2`): - Diffusion perf: include `"diffusion"` in the `mark` array - HunyuanImage local-weight cases: add `"local_model"` to the `mark` array -**Parametrization IDs** +#### Parametrization IDs Each `(server, benchmark index)` pair becomes one `pytest.param` with id `{test_name}-{suffix}`. The suffix comes from `benchmark_params[].name` when set (for example `test_tts-p0`, `test_omni-p1`); otherwise it is derived from fields like `task` / `eval_phase`. -**Benchmark result filenames** +#### Benchmark result filenames Result files use the **runtime** hardware label from `get_runtime_resource_label()` (detected GPU/NPU name on the machine that ran the job), **not** `mark.hardware_marks.res`. On the default H100 CI pool, `H100` is **omitted** from filenames (`resource_label_for_filename`). @@ -132,7 +135,7 @@ Examples: - Omni/TTS: `result_{test_name}_{optional_hw}_{dataset}_....json` under `BENCHMARK_DIR` - Diffusion: one aggregate `diffusion_result_{config_stem}_{optional_hw}_{timestamp}.json` per source JSON file (array of all runs from that file) -**Local commands** +#### Local commands ```bash # Bulk load + filter by JSON mark @@ -146,18 +149,19 @@ pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json ``` + See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). -**`server_params` Configuration** +#### `server_params` Configuration -*Basic Parameters* +##### Basic Parameters -| Parameter | Required | Example | Description | -| ----------------- | -------- | ---------------------------------- | ----------------------------- | -| model | Yes | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path | -| stage_config_name | Yes | "qwen3_omni_moe.yaml" | Deploy configuration file name | +| Parameter | Required | Example | Description | +| ----------------- | -------- | ------------------------- | ------------------------------ | +| model | Yes | Qwen/Qwen3-Omni-30B-A3B… | Model name or path | +| stage_config_name | Yes | qwen3_omni_moe.yaml | Deploy configuration file name | -*Dynamic Configuration (update/delete)* +##### Dynamic Configuration (update/delete) Supports incremental modifications based on the basic configuration: @@ -166,9 +170,9 @@ Supports incremental modifications based on the basic configuration: | update | Update or add configuration items | | delete | Delete specified configuration items | -**Example**: +##### Example -``` +```json "update": { "async_chunk": true, // Enable asynchronous chunk processing "stages": { @@ -184,20 +188,22 @@ Supports incremental modifications based on the basic configuration: } ``` -**`benchmark_params` Configuration** +#### `benchmark_params` Configuration You can add any benchmark running parameters you need here. For all optional parameters, refer to the [benchmark documentation](https://github.com/vllm-project/vllm-omni/blob/main/docs/cli/bench/serve.md). General modifications are as follows: -1. Change the --xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file. -2. For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file. -3. Optionally add a `baseline` object (see **Baseline thresholds** below). If you omit `baseline` or leave it empty, the performance test still runs but does not assert metric thresholds from this field. -4. Set `"name"` on each `benchmark_params` entry for stable pytest ids and readable result keys. -5. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: +1. Change the --xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file. +2. For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file. +3. Optionally add a `baseline` object (see **Baseline thresholds** below). If you omit `baseline` or leave it empty, the performance test still runs but does not assert metric thresholds from this field. +4. Set `"name"` on each `benchmark_params` entry for stable pytest ids and readable result keys. +5. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: -| Parameter | Type | Required | Example/Values | Description | -| --------------- | ----------- | -------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| name | string | No | `"1024x1024_steps4"` | Stable suffix for pytest param id and logs. | -| num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | -| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | -| max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | -| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. | +| Parameter | Type | Required | Example/Values | Description | +| --------------- | ------------- | -------- | -------------------- | ------------------------------------ | +| name | string | No | `"1024x1024_steps4"` | Stable suffix for pytest param id | +| num_prompts | int / array | Yes | 10, [10, 20, 30] | Number of requests (scalar or sweep) | +| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second (QPS mode) | +| max_concurrency | int / array | No | 1, [1, 2, 3] | Max in-flight requests | +| baseline | object | No | see below | Optional hardware-nested thresholds | + +`num_prompts`, `request_rate`, and `max_concurrency` accept scalars or arrays; single values expand to match sweep length. QPS and concurrency modes are mutually exclusive. `baseline` top-level keys must be hardware pytest markers from `pyproject.toml`; metric names and values are free-form (scalars or sweep-aligned lists). Omit `baseline` to run without threshold assertions. diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index ff53cbbf7..533c9221d 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -1,13 +1,20 @@ # Test Execution Guide + ## Setting Up the Test Environment + ### Creating a Container + vLLM-Omni provides an official Docker image for deployment. These images are built upon vLLM Docker images and are available on [Docker Hub](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which vLLM release it is based on. For a local test environment, you can follow the steps below to create a container: + ## Installing Dependencies + ### vLLM & vLLM-Omni + vLLM-Omni is built based on vLLM. You can follow [install guide](../../getting_started/installation/README.md) to build your local environment. ### Test Case Dependencies + When running test cases, you may need to install the following dependencies: ```bash @@ -16,6 +23,7 @@ apt-get install -y espeak-ng jq ``` ## Running Tests + Our test scripts use the pytest framework. First, please use `git clone https://github.com/vllm-project/vllm-omni.git` to download the vllm-omni source code. Then, in the root directory of vllm-omni, you can run the following commands in your local test environment to execute the corresponding test cases. ### CI job runners (L2–L4): logs, timing, and timeouts @@ -25,7 +33,7 @@ Our test scripts use the pytest framework. First, please use `git clone https:// **Log layout** (default under `logs/ready_jobs`, `logs/merge_jobs`, or `logs/nightly_jobs`; override with `--log-dir`): | Path | Description | -|------|-------------| +| ------ | ------------- | | `jobs/*.sh` | Generated per-step bash wrappers | | `jobs/.job_timeouts` | Job key → `timeout_in_minutes` from Buildkite YAML (when set) | | `.log` | Tee output for each job | @@ -33,7 +41,7 @@ Our test scripts use the pytest framework. First, please use `git clone https:// **Timing:** while each job runs, stderr prints `finished in …` or `failed after …`. When all jobs finish, a summary is printed and written to `timing_summary.log`, for example: -``` +```text === Job timing summary === TTS_VoxCPM2_Test 25m 40s OK Omni_Qwen3-Omni_Test 8m 05s TIMED OUT @@ -114,7 +122,6 @@ Failed jobs: 1/2 pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model ``` - === "L4 level" **Recommended:** run CI-aligned nightly jobs from the repo root via [`tools/nightly/run_nightly_jobs.sh`](https://github.com/vllm-project/vllm-omni/blob/main/tools/nightly/run_nightly_jobs.sh). The script reads [`.buildkite/cuda/test-nightly.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml), generates per-step bash wrappers, runs pytest (perf jobs first, then aggregates perf JSON into Excel), and tees logs under `logs/nightly_jobs/` (see [CI job runners](#ci-job-runners-l2l4-logs-timing-and-timeouts); `generate_nightly_perf_excel` is included in the timing summary). Requires `bash`, `python3`, and PyYAML (`pip install pyyaml`). @@ -160,7 +167,7 @@ Failed jobs: 1/2 pytest -sv tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json ``` - Nightly **Perf Test** jobs in ``test-nightly.yml`` use ``--test-config-file`` only (no ``-m``). E2e L4 function tests still use ``full_model`` + ``--run-level full_model`` (see [test-nightly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml)). Example: + Nightly **Perf Test** jobs in [``test-nightly.yml``](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-nightly.yml) use ``--test-config-file`` only (no ``-m``). Weekly **Perf Test** in [``test-weekly.yml``](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml) runs ``test_qwen3_omni_vllm_text.json`` and ``test_qwen3_omni_multi_replicas.json`` (JSON ``mark`` includes ``slow``). E2e L4 function tests use ``full_model`` + ``--run-level full_model``. Example: === "L5 level" @@ -186,9 +193,13 @@ Failed jobs: 1/2 ``` - The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). + The latest L5 CI jobs are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml): + + - **Reliability** (+ invalid-parameter), **Perf Test** (Omni vLLM Text / Multi-Replica), and **Simple · CPU Coverage Test** → `WEEKLY=1` (or PR label `weekly-test` for Reliability/Perf). Scheduled upload of those env-gated steps still requires `main`. + - **E2E Tests** (slow Omni / TTS / Diffusion sweeps) → `NON_CRITICAL=1` (pipeline upload also accepts `WEEKLY=1` / `weekly-test`; the group `if` still needs `NON_CRITICAL`) You can find more information about markers in [Test Writing Guide — Markers for Tests](./test_writing_guide.md#markers-for-tests). ## Adding New Test Cases + Please refer to the [Test System Overview](./test_system_overview.md) and [Test Writing Guide](./test_writing_guide.md). diff --git a/docs/contributing/ci/test_system_overview.md b/docs/contributing/ci/test_system_overview.md index 335ed5c53..b4d3ce121 100644 --- a/docs/contributing/ci/test_system_overview.md +++ b/docs/contributing/ci/test_system_overview.md @@ -12,8 +12,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Level Scope & Focus Model Coverage Strategy - Feature Coverage Strategy - Interface Coverage Strategy + Module Strategy Tags Time Cost Test Dir @@ -31,7 +30,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari / / / - / PR Checklist / / @@ -43,7 +41,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari / / / - / CI Failures / / @@ -53,7 +50,6 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Unit tests for components like entrypoints, models / / - / core_model and cpu <15min @@ -71,9 +67,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L2
(E2E across models & GPU-required UT) Online (basic deployment scenarios):
dummy, normal inference function (output format, stream), some instance startup UT - High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy) - High-priority features and component tests that require launching instances - High-priority interfaces (using random lightweight models) + Key models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy) + Key features and modules that require launching instances (prefer random weights) core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion Model E2E:
@@ -95,9 +90,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L3
(Important Perf & Integration & Accuracy) Online & Offline (multiple deployment scenarios):
real model, normal inference function, normal accuracy - High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy - Medium-priority features and component tests that require launching instances - Medium-priority interfaces (using random lightweight models) + Key models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy + Key features and modules that require launching instances (using real weights) advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion <30min @@ -120,9 +114,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari L4
(Perf & Integration & Accuracy) Online: full functional scenarios + performance test + doc test + accuracy test - High-priority models: function, performance, accuracy, and doc testing
Medium-priority models: function and doc testing - Low-priority features and component tests that require launching instances - Low-priority interfaces (using real weights) + Key models: function, performance, accuracy, and doc testing + Other features and modules that require launching instances (using real weights) full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion <3 hour @@ -147,11 +140,10 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari GPU - L5
(Stability & Reliability) - Online: long-term stability test + reliability test - Long-term stability and reliability testing for high-priority models
Low-priority models: function and doc testing + L5
(Stability & Reliability & selected Perf) + Online: long-term stability + reliability; coverage test + Long-term stability and reliability testing for key models
Non-critical scenario performance and accuracy testing for key models
Other models: function and doc testing / - Invalid-parameter validation for high-priority interfaces slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion Depends on reality @@ -178,8 +170,8 @@ For per-level test authoring (markers, examples), see [Test Writing Guide](./tes Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues. -1. ***PR Checklist ([`.github/PULL_REQUEST_TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/.github/PULL_REQUEST_TEMPLATE.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance. -2. ***CI Failure Explanation ([CI Failures](./failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency. +1. ***PR Checklist ([`.github/PULL_REQUEST_TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/.github/PULL_REQUEST_TEMPLATE.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance. +2. ***CI Failure Explanation ([CI Failures](./failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency. ## Notes diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 7912fc4a1..33c633f9d 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -509,7 +509,7 @@ L4 level testing is a comprehensive quality audit before a version release. It e #### 3.2 Testing Content and Scope - ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. -- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. +- ***Performance Testing***: JSON configs under `tests/dfx/perf/tests/` drive Omni, TTS, and diffusion serving benchmarks via `run_benchmark.py` or `run_diffusion_benchmark.py`, measuring throughput, latency, and related metrics under configured workloads. - ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. #### 3.3 Test Directory and Execution Files @@ -567,7 +567,7 @@ L5 level testing focuses on the performance of model services under ***long-runn - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. - ***Run Command***: - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). Weekly CI (`.buildkite/cuda/test-weekly.yml`) runs one step per suite (`WEEKLY=1` or PR label `weekly-test`) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). - ***Script Example***:
diff --git a/tests/buildkite/test_upload_pipeline.py b/tests/buildkite/test_upload_pipeline.py index bd0f50f5c..c994f8dd2 100644 --- a/tests/buildkite/test_upload_pipeline.py +++ b/tests/buildkite/test_upload_pipeline.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import sys @@ -61,16 +62,23 @@ def test_bootstrap_steps_loaded_from_file() -> None: def test_docs_only_allows_main_scheduled_nightly_weekly_only() -> None: - """skip_all: no PR labels; main + NIGHTLY=1 / WEEKLY=1 still gates scheduled CI.""" + """skip_all: no PR labels; main + NIGHTLY=1 / WEEKLY=1 / NON_CRITICAL=1 still gates scheduled CI.""" rendered = _render(["docs/foo.md"]) assert "key: image-build" in rendered assert "key: upload-nightly-pipeline" in rendered assert "key: upload-weekly-pipeline" in rendered - # Scheduled nightly also uploads L2/L3 with --e2e - assert "key: upload-ready-pipeline" in rendered - assert "key: upload-merge-pipeline" in rendered - assert 'build.env("NIGHTLY") == "1"' in rendered + # Scheduled main+WEEKLY=1 uploads L2/L3 with --e2e; NIGHTLY still gates L4 only. + doc = yaml.safe_load(rendered) + by_key = {step["key"]: step for step in doc["steps"]} + assert "NIGHTLY" not in by_key["upload-ready-pipeline"]["if"] + assert 'build.branch == "main"' in by_key["upload-ready-pipeline"]["if"] + assert 'build.env("WEEKLY") == "1"' in by_key["upload-ready-pipeline"]["if"] + assert "NIGHTLY" not in by_key["upload-merge-pipeline"]["if"] + assert 'build.branch == "main"' in by_key["upload-merge-pipeline"]["if"] + assert 'build.env("WEEKLY") == "1"' in by_key["upload-merge-pipeline"]["if"] + assert 'build.env("NIGHTLY") == "1"' in by_key["upload-nightly-pipeline"]["if"] assert 'build.env("WEEKLY") == "1"' in rendered + assert 'build.env("NON_CRITICAL") == "1"' in rendered assert "nightly-test" not in rendered assert "weekly-test" not in rendered assert "merge-test" not in rendered @@ -78,6 +86,19 @@ def test_docs_only_allows_main_scheduled_nightly_weekly_only() -> None: assert "if: false" not in rendered +def test_npu_docs_only_does_not_upload_ready_on_nightly() -> None: + """NPU skip_all: scheduled NIGHTLY still uploads L4, not L2 ready.""" + rendered = _render_bootstrap_pipeline( + BOOTSTRAP_STEPS_TEMPLATE, + decision=resolve_ci_decision(["docs/foo.md"]), + path=Path(".buildkite/npu/bootstrap-upload-steps.yml"), + ) + doc = yaml.safe_load(rendered) + by_key = {step["key"]: step for step in doc["steps"]} + assert "upload-ready-pipeline" not in by_key + assert 'build.env("NIGHTLY") == "1"' in by_key["upload-nightly-pipeline"]["if"] + + def test_yaml_gated_l45_only_does_not_unconditionally_build_image() -> None: rendered = _render([".buildkite/cuda/test-nightly.yml"]) assert "if: true" not in rendered diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index 55ccf14ba..5453d6a90 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -10,7 +10,7 @@ "num_cards": 3 } }, - "full_model", + "slow", "omni" ], "server_params": { diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index f24bd957f..d077188ad 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -10,7 +10,7 @@ "num_cards": 2 } }, - "full_model", + "slow", "omni" ], "server_params": { diff --git a/tests/cosmos3/test_session_memory_equivalence.py b/tests/diffusion/models/cosmos3/test_session_memory_equivalence.py similarity index 100% rename from tests/cosmos3/test_session_memory_equivalence.py rename to tests/diffusion/models/cosmos3/test_session_memory_equivalence.py diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index fdcef37b5..099287701 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ Quantization quality gate for diffusion models. @@ -53,6 +53,7 @@ _BENCH_MODULE_NAME = "benchmarks.diffusion.quantization_quality" if _BENCH_MODULE_NAME not in sys.modules: _spec = importlib.util.spec_from_file_location(_BENCH_MODULE_NAME, _BENCH_MODULE_PATH) + assert _spec is not None and _spec.loader is not None _mod = importlib.util.module_from_spec(_spec) sys.modules[_BENCH_MODULE_NAME] = _mod _spec.loader.exec_module(_mod) @@ -93,7 +94,7 @@ class QualityTestConfig: return self.quantized_model return self.model or "" - def quantization_ref(self) -> str | None: + def quantization_ref(self) -> str | dict[str, object] | None: if self.quantized_model is not None: return None return self.quantization @@ -500,6 +501,8 @@ def _quality_param(c: QualityTestConfig): marks.append( pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).") ) + if c.id == "fp8_ltx2": + marks.append(pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/6245")) return pytest.param(c, id=c.id, marks=marks) diff --git a/tests/examples/offline_inference/test_image_task_prompts.py b/tests/examples/offline_inference/test_image_task_prompts.py index 57e8f1560..421082ed6 100644 --- a/tests/examples/offline_inference/test_image_task_prompts.py +++ b/tests/examples/offline_inference/test_image_task_prompts.py @@ -3,15 +3,27 @@ from __future__ import annotations +import functools +import importlib.util + import pytest from PIL import Image -from examples.offline_inference.image_to_video.image_to_video import build_image_to_video_prompt -from examples.offline_inference.text_to_image.text_to_image import build_text_to_image_prompt +from tests.examples.helpers import EXAMPLES pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +@functools.cache +def _load_example_module(relative_path: str, module_name: str): + path = EXAMPLES / relative_path + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @pytest.mark.parametrize( ("negative_prompt", "expected_negative_prompt"), [(None, None), ("", ""), ("blurry", "blurry")], @@ -20,7 +32,11 @@ def test_text_to_image_builds_canonical_prompt( negative_prompt: str | None, expected_negative_prompt: str | None, ) -> None: - result = build_text_to_image_prompt("a red fox", negative_prompt) + mod = _load_example_module( + "offline_inference/text_to_image/text_to_image.py", + "text_to_image_example", + ) + result = mod.build_text_to_image_prompt("a red fox", negative_prompt) assert result["prompt"] == "a red fox" assert result["modalities"] == ["image"] @@ -31,9 +47,13 @@ def test_text_to_image_builds_canonical_prompt( def test_image_to_video_builds_canonical_prompt() -> None: + mod = _load_example_module( + "offline_inference/image_to_video/image_to_video.py", + "image_to_video_example", + ) image = Image.new("RGB", (32, 16), "red") - result = build_image_to_video_prompt( + result = mod.build_image_to_video_prompt( prompt="the fox turns toward the camera", negative_prompt="flicker", media_inputs={"image": image}, diff --git a/tests/examples/offline_inference/test_text_to_image.py b/tests/examples/offline_inference/test_text_to_image.py index 701f1a184..5ee2a2fae 100644 --- a/tests/examples/offline_inference/test_text_to_image.py +++ b/tests/examples/offline_inference/test_text_to_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Offline inference tests: text-to-image. See examples/offline_inference/text_to_image/README.md @@ -26,6 +29,8 @@ EXAMPLE_OUTPUT_SUBFOLDER = "example_offline_t2i" def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool, str]: if h2_title == "Web UI Demo": return True, f"README section '{h2_title}' is intentionally excluded for examples tests" + if "krea/Krea-2-Raw" in code: + return True, "krea/Krea-2-Raw is gated on Hugging Face Hub; CI has no access" return False, "" diff --git a/tests/examples/offline_inference/test_text_to_video_prompts.py b/tests/examples/offline_inference/test_text_to_video_prompts.py index ff7244d65..342638ccc 100644 --- a/tests/examples/offline_inference/test_text_to_video_prompts.py +++ b/tests/examples/offline_inference/test_text_to_video_prompts.py @@ -3,19 +3,27 @@ from __future__ import annotations +import functools +import importlib.util + import pytest import torch -from examples.offline_inference.text_to_video.text_to_video import ( - _MODEL_PRESETS, - _detect_preset, - _normalize_float_tensor, - build_text_to_video_prompt, -) +from tests.examples.helpers import EXAMPLES pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +@functools.cache +def _load_text_to_video(): + path = EXAMPLES / "offline_inference" / "text_to_video" / "text_to_video.py" + spec = importlib.util.spec_from_file_location("text_to_video_example", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @pytest.mark.parametrize( ("negative_prompt", "expected_negative_prompt"), [(None, None), ("", ""), ("blurry", "blurry")], @@ -24,7 +32,8 @@ def test_text_to_video_builds_canonical_prompt( negative_prompt: str | None, expected_negative_prompt: str | None, ) -> None: - result = build_text_to_video_prompt("a robot waves", negative_prompt) + mod = _load_text_to_video() + result = mod.build_text_to_video_prompt("a robot waves", negative_prompt) assert result["prompt"] == "a robot waves" assert result["modalities"] == ["video"] @@ -50,11 +59,13 @@ def test_detect_preset_is_scoped_to_model_family( model_class_name: str | None, preset_name: str, ) -> None: - assert _detect_preset(model, model_class_name) is _MODEL_PRESETS[preset_name] + mod = _load_text_to_video() + assert mod._detect_preset(model, model_class_name) is mod._MODEL_PRESETS[preset_name] def test_lingbot_preset_matches_lingbot_defaults() -> None: - assert _MODEL_PRESETS["lingbot"] == { + mod = _load_text_to_video() + assert mod._MODEL_PRESETS["lingbot"] == { "model_class_name": "LingBotVideoPipeline", "height": 192, "width": 320, @@ -68,24 +79,27 @@ def test_lingbot_preset_matches_lingbot_defaults() -> None: def test_normalize_float_tensor_preserves_zero_to_one_frames() -> None: + mod = _load_text_to_video() frames = torch.tensor([0.0, 0.25, 1.0, 1.5]) - assert torch.equal(_normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0])) + assert torch.equal(mod._normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0])) def test_normalize_float_tensor_maps_negative_one_to_one_frames() -> None: + mod = _load_text_to_video() frames = torch.tensor([-1.5, -1.0, 0.0, 1.0, 1.5]) assert torch.equal( - _normalize_float_tensor(frames, "negative_one_to_one"), + mod._normalize_float_tensor(frames, "negative_one_to_one"), torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]), ) def test_normalize_float_tensor_uses_one_contract_for_ambiguous_frames() -> None: + mod = _load_text_to_video() frames = [torch.tensor([0.0, 0.5, 1.0]), torch.tensor([-1.0, 0.0, 1.0])] - normalized = [_normalize_float_tensor(frame, "negative_one_to_one") for frame in frames] + normalized = [mod._normalize_float_tensor(frame, "negative_one_to_one") for frame in frames] assert torch.equal(normalized[0], torch.tensor([0.5, 0.75, 1.0])) assert torch.equal(normalized[1], torch.tensor([0.0, 0.5, 1.0])) diff --git a/tests/test_indextts2_optional_dependencies.py b/tests/model_executor/models/indextts2/test_optional_dependencies.py similarity index 99% rename from tests/test_indextts2_optional_dependencies.py rename to tests/model_executor/models/indextts2/test_optional_dependencies.py index cc3c46b63..24c6131fd 100644 --- a/tests/test_indextts2_optional_dependencies.py +++ b/tests/model_executor/models/indextts2/test_optional_dependencies.py @@ -14,7 +14,7 @@ from packaging.requirements import Requirement pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -ROOT = Path(__file__).parents[1] +ROOT = Path(__file__).parents[4] TEXT_PROCESSING_PATH = ROOT / "vllm_omni" / "model_executor" / "models" / "indextts2" / "text_processing_v2_5.py" INDEXTTS2_FRONTEND_PACKAGES = { "fugashi",