From 0b3898c88433d915da632eafe1cdbabe4f83a264 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:47:20 +0800 Subject: [PATCH] [CI] Point merge/nightly Buildkite triggers at minicpm-challenge (#5497) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/amd/scripts/bootstrap-amd-omni.sh | 4 + .buildkite/common/scripts/skip_ci.py | 10 +- .buildkite/common/scripts/upload_pipeline.py | 23 +- .buildkite/cuda/bootstrap-upload-steps.yml | 10 +- .buildkite/cuda/test-merge.yml | 453 ++++----- .buildkite/cuda/test-nightly.yml | 936 +++++++++--------- .buildkite/cuda/test-ready.yml | 547 +++++----- .../intel/scripts/bootstrap-intel-omni.sh | 4 + .buildkite/npu/test-npu-nightly.yml | 80 +- .buildkite/npu/test-npu-ready.yml | 34 +- .../offline_inference/test_minicpmo_4_5.py | 2 - tests/e2e/online_serving/test_minicpmo_4_5.py | 2 - 12 files changed, 982 insertions(+), 1123 deletions(-) diff --git a/.buildkite/amd/scripts/bootstrap-amd-omni.sh b/.buildkite/amd/scripts/bootstrap-amd-omni.sh index 32097cb8c..377723223 100755 --- a/.buildkite/amd/scripts/bootstrap-amd-omni.sh +++ b/.buildkite/amd/scripts/bootstrap-amd-omni.sh @@ -6,6 +6,10 @@ set -euo pipefail +# minicpm-challenge branch: skip entire AMD CI (no pipeline upload => no image build / tests). +echo "--- Skipping AMD CI on this branch (including image build)" +exit 0 + source .buildkite/common/scripts/resolve_skip_ci.sh if [[ -z "${RUN_ALL:-}" ]]; then diff --git a/.buildkite/common/scripts/skip_ci.py b/.buildkite/common/scripts/skip_ci.py index b452b5adf..f87170979 100644 --- a/.buildkite/common/scripts/skip_ci.py +++ b/.buildkite/common/scripts/skip_ci.py @@ -282,7 +282,7 @@ def _git(*args: str) -> subprocess.CompletedProcess[str]: def _resolve_diff_range() -> str | None: - """Return a git diff range for PR or main builds, or None when unavailable.""" + """Return a git diff range for PR or push-branch builds, or None when unavailable.""" is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get( "BUILDKITE_PULL_REQUEST", "" ) @@ -302,13 +302,15 @@ def _resolve_diff_range() -> str | None: return None return f"{base_ref}...{commit}" - if os.environ.get("BUILDKITE_BRANCH", "") == "main": + # main: ready/nightly/weekly push CI; minicpm-challenge: merge (L3) push CI + branch = os.environ.get("BUILDKITE_BRANCH", "") + if branch in ("main", "minicpm-challenge"): if _git("rev-parse", "--verify", f"{commit}^").returncode != 0: - _log("main commit has no parent; using safe defaults") + _log(f"{branch} commit has no parent; using safe defaults") return None return f"{commit}^..{commit}" - _log("not PR/main build; using safe defaults") + _log("not PR/main/minicpm-challenge build; using safe defaults") return None diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index b27f78c3a..73eda47ab 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -59,11 +59,12 @@ E2E_GROUP_MARKER = "E2E Test" CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" CUDA_NIGHTLY_ONLY = ( - '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")' + '(build.pull_request.labels includes "nightly-test") || ' + '(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1")' ) NPU_NIGHTLY_ONLY = ( - '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' - '(build.branch != "main" && (' + '(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || ' + '(build.branch != "minicpm-challenge" && (' 'build.pull_request.labels includes "nightly-test" || ' 'build.pull_request.labels includes "omni-test" || ' 'build.pull_request.labels includes "tts-test" || ' @@ -103,7 +104,8 @@ def _format_bootstrap_if(expr: str) -> str: def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: disabled = "false" - nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"' + # L4 nightly scheduled push CI runs on minicpm-challenge (not main). + nightly_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1"' nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY if platform == "npu": @@ -118,11 +120,12 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: merge_base = disabled else: ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' - merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"' - merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"' + # L3 merge push CI runs on minicpm-challenge (not main). + merge_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"' + merge_pr = 'build.branch != "minicpm-challenge" && build.pull_request.labels includes "merge-test"' nightly_label_if = ( - '(build.branch == "main" && build.env("NIGHTLY") == "1") || ' - '(build.branch != "main" && (' + '(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || ' + '(build.branch != "minicpm-challenge" && (' 'build.pull_request.labels includes "nightly-test" || ' 'build.pull_request.labels includes "omni-test" || ' 'build.pull_request.labels includes "tts-test" || ' @@ -140,8 +143,8 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"' if decision.skip_all: - # Docs / skip-mark only: no PR-label escape hatch. Main scheduled - # NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5. + # Docs / skip-mark only: no PR-label escape hatch. Scheduled + # NIGHTLY=1 (minicpm-challenge) still runs L4 + L2/L3 (--e2e); WEEKLY=1 (main) still runs L5. image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main ready_expr = nightly_main merge_expr = nightly_main if platform == "cuda" else disabled diff --git a/.buildkite/cuda/bootstrap-upload-steps.yml b/.buildkite/cuda/bootstrap-upload-steps.yml index f2467bf42..538ee78f0 100644 --- a/.buildkite/cuda/bootstrap-upload-steps.yml +++ b/.buildkite/cuda/bootstrap-upload-steps.yml @@ -8,35 +8,35 @@ steps: agents: queue: "cpu_queue_premerge" - # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + # L2 Test — PR ready label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group) - label: "Upload Ready Pipeline" depends_on: image-build key: upload-ready-pipeline commands: - | EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then EXTRA="--e2e" fi python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml agents: queue: "cpu_queue_premerge" - # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group) + # L3 Test — PR merge-test label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group) - label: "Upload Merge Pipeline" depends_on: image-build key: upload-merge-pipeline commands: - | EXTRA="" - if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then + if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then EXTRA="--e2e" fi python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" - # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) + # L4 Test — minicpm-challenge+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild) - label: "Upload Nightly Pipeline" depends_on: image-build key: upload-nightly-pipeline diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 88bd2fb3e..5074c7d5f 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -18,282 +18,227 @@ steps: - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - - group: ":card_index_dividers: Diffusion Test" - depends_on: upload-merge-pipeline - steps: - - label: "Diffusion · LoRA Test" - timeout_in_minutes: 30 - commands: - - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" - mirror_hardwares: l4_1 + # - group: ":card_index_dividers: Diffusion Test" + # depends_on: upload-merge-pipeline + # steps: + # - label: "Diffusion · LoRA Test" + # timeout_in_minutes: 30 + # commands: + # - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" + # mirror_hardwares: l4_1 - - label: "Diffusion · Offloader Test" - timeout_in_minutes: 40 - commands: - - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda" - mirror_hardwares: l4_1 + # - label: "Diffusion · Offloader Test" + # timeout_in_minutes: 40 + # commands: + # - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda" + # mirror_hardwares: l4_1 - - label: "Distributed Test" - depends_on: upload-merge-pipeline - timeout_in_minutes: 60 - commands: - - | - timeout 55m bash -c " - set -e - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model' - " - mirror_hardwares: h100_1 + # - label: "Distributed Test" + # depends_on: upload-merge-pipeline + # timeout_in_minutes: 60 + # commands: + # - | + # timeout 55m bash -c " + # set -e + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model' + # " + # mirror_hardwares: h100_1 - - group: ":card_index_dividers: Entrypoints Test" - depends_on: upload-merge-pipeline - steps: - - label: "Entrypoint Test with H100" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" - mirror_hardwares: h100_2 + # - group: ":card_index_dividers: Entrypoints Test" + # depends_on: upload-merge-pipeline + # steps: + # - label: "Entrypoint Test with H100" + # timeout_in_minutes: 60 + # commands: + # - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" + # mirror_hardwares: h100_2 - - label: "Entrypoint Test with L4" - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" - mirror_hardwares: l4_1 + # - label: "Entrypoint Test with L4" + # timeout_in_minutes: 30 + # commands: + # - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model" + # mirror_hardwares: l4_1 - - group: ":card_index_dividers: E2E Test" - depends_on: upload-merge-pipeline - steps: - - label: "TTS · Qwen3-TTS CustomVoice Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - commands: - - | - timeout 40m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 + # - group: ":card_index_dividers: E2E Test" + # depends_on: upload-merge-pipeline + # steps: + # - label: "TTS · Qwen3-TTS CustomVoice Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen3_tts_customvoice.py + # - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py + # - vllm_omni/model_executor/models/qwen3_tts/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py + # - vllm_omni/deploy/qwen3_tts.yaml + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + # pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · Qwen3-TTS Base Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_base.py - - tests/e2e/offline_inference/test_qwen3_tts_base.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - commands: - - | - timeout 40m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · Qwen3-TTS Base Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen3_tts_base.py + # - tests/e2e/offline_inference/test_qwen3_tts_base.py + # - vllm_omni/model_executor/models/qwen3_tts/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py + # - vllm_omni/deploy/qwen3_tts.yaml + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + # pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · Higgs-Audio-V3 Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_higgs_audio_v3.py + # - vllm_omni/model_executor/models/higgs_audio_v3/ + # - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py + # - vllm_omni/deploy/higgs_multimodal_qwen3.yaml + # commands: + # - | + # timeout 20m bash -c " + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # export VLLM_USE_DEEP_GEMM=0 + # export VLLM_MOE_USE_DEEP_GEMM=0 + # pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model' + # " + # mirror_hardwares: l4_1 - - label: "Omni · Qwen3-Omni Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_qwen3_omni.py - - tests/e2e/online_serving/test_qwen3_omni.py - - vllm_omni/model_executor/models/qwen3_omni/ - - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - - vllm_omni/deploy/qwen3_omni_moe.yaml - timeout_in_minutes: 50 - commands: - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_2 + # - label: "Omni · Qwen3-Omni Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_qwen3_omni.py + # - tests/e2e/online_serving/test_qwen3_omni.py + # - vllm_omni/model_executor/models/qwen3_omni/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py + # - vllm_omni/deploy/qwen3_omni_moe.yaml + # timeout_in_minutes: 50 + # commands: + # - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_minicpmo_4_5.py - - tests/e2e/online_serving/test_minicpmo_4_5.py - - vllm_omni/deploy/minicpmo_4_5.yaml timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model' - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Duplex Test" - source_file_dependencies: - - tests/assets/minicpmo_4_5/response_required_16k.wav - - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav - - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py - - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py - - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py - - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py - - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py - - tests/helpers/minicpmo_4_5_duplex.py - - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - - vllm_omni/experimental/fullduplex/ - - vllm_omni/model_executor/models/minicpmo_4_5/ - - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - - vllm_omni/core/sched/ - - vllm_omni/outputs/multimodal_accumulation.py - - vllm_omni/engine/stage_init_utils.py - - vllm_omni/engine/async_omni_engine.py - - vllm_omni/entrypoints/cli/serve.py timeout_in_minutes: 50 commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model' mirror_hardwares: h100_2 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_voxcpm2_tts.py - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · VoxCPM2 Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_voxcpm2_tts.py + # - tests/e2e/online_serving/test_voxcpm2_tts.py + # - vllm_omni/model_executor/models/voxcpm2/ + # - vllm_omni/deploy/voxcpm2.yaml + # timeout_in_minutes: 30 + # commands: + # - | + # timeout 30m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · SoulX-Singer Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_soulxsinger.py - - vllm_omni/diffusion/models/soulx_singer/ - - vllm_omni/deploy/soulxsinger_svc.yaml - - vllm_omni/deploy/soulxsinger_svs.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · SoulX-Singer Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_soulxsinger.py + # - vllm_omni/diffusion/models/soulx_singer/ + # - vllm_omni/deploy/soulxsinger_svc.yaml + # - vllm_omni/deploy/soulxsinger_svs.yaml + # timeout_in_minutes: 30 + # commands: + # - | + # timeout 30m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model' + # " + # mirror_hardwares: l4_1 - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_bagel.py - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - timeout_in_minutes: 20 - commands: - - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model' - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Edit Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image_edit.py + # - vllm_omni/diffusion/models/qwen_image/ + # timeout_in_minutes: 20 + # commands: + # - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # mirror_hardwares: h100_1 + # - label: "Diffusion · Bagel Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_bagel.py + # - tests/e2e/online_serving/test_bagel.py + # - vllm_omni/model_executor/models/bagel/ + # - vllm_omni/diffusion/models/bagel/ + # - vllm_omni/model_executor/stage_input_processors/bagel.py + # - vllm_omni/deploy/bagel.yaml + # timeout_in_minutes: 20 + # commands: + # - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c ' - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model' - ' - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image.py + # - vllm_omni/diffusion/models/qwen_image/ + # commands: + # - | + # timeout 40m bash -c ' + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # ' + # mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Layered Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image_layered.py + # - vllm_omni/diffusion/models/qwen_image/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_wan22_t2v.py - - tests/e2e/online_serving/test_wan22_t2v.py - - vllm_omni/diffusion/models/wan2_2/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Wan22 Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_wan22_t2v.py + # - tests/e2e/online_serving/test_wan22_t2v.py + # - vllm_omni/diffusion/models/wan2_2/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Z Image Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_z_image.py - - tests/e2e/online_serving/test_z_image.py - - vllm_omni/diffusion/models/z_image/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 - - label: "Diffusion · FLUX.2 Klein Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_flux2_klein.py - - vllm_omni/diffusion/models/flux2_klein/ - timeout_in_minutes: 30 - commands: - - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - mirror_hardwares: l4_1 + # - label: "Diffusion · Z Image Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_z_image.py + # - tests/e2e/online_serving/test_z_image.py + # - vllm_omni/diffusion/models/z_image/ + # timeout_in_minutes: 30 + # commands: + # - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + # mirror_hardwares: l4_1 + # - label: "Diffusion · FLUX.2 Klein Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/test_flux2_klein.py + # - vllm_omni/diffusion/models/flux2_klein/ + # timeout_in_minutes: 30 + # commands: + # - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + # mirror_hardwares: l4_1 diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 836bc5e3b..978277873 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -24,517 +24,517 @@ steps: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model" mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Doc Test with L4" - timeout_in_minutes: 90 - commands: - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" - mirror_hardwares: l4_4 + # - label: ":full_moon: Omni · Doc Test with L4" + # timeout_in_minutes: 90 + # commands: + # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + # - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" + # mirror_hardwares: l4_4 - - label: ":full_moon: Omni · Doc Test with H100" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Omni · Doc Test with H100" + # timeout_in_minutes: 90 + # commands: + # - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Accuracy Test" - timeout_in_minutes: 180 - commands: - - export SEED_TTS_WER_EVAL=1 - - export SEED_TTS_EVAL_DEVICE=cuda:1 - - | - set +e - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model - EXIT=$$? - buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" - exit $$EXIT - mirror_hardwares: h100_2 + # - label: ":full_moon: Omni · Accuracy Test" + # timeout_in_minutes: 180 + # commands: + # - export SEED_TTS_WER_EVAL=1 + # - export SEED_TTS_EVAL_DEVICE=cuda:1 + # - | + # set +e + # pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model + # EXIT=$$? + # buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json" + # exit $$EXIT + # mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Perf Test · No Async Chunk" - key: nightly-omni-performance-no-async-chunk - timeout_in_minutes: 300 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_2 + # - label: ":full_moon: Omni · Perf Test · No Async Chunk" + # key: nightly-omni-performance-no-async-chunk + # timeout_in_minutes: 300 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Perf Test · Async Chunk" - key: nightly-omni-performance-async-chunk - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_2 + # - label: ":full_moon: Omni · Perf Test · Async Chunk" + # key: nightly-omni-performance-async-chunk + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Perf Test · vLLM Text" - key: nightly-omni-performance-vllm-text - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_2 + # - label: ":full_moon: Omni · Perf Test · vLLM Text" + # key: nightly-omni-performance-vllm-text + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_2 - - label: ":full_moon: Omni · Perf Test · Multi-Replica" - key: nightly-omni-performance-multi-replicas - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_3 + # - label: ":full_moon: Omni · Perf Test · Multi-Replica" + # key: nightly-omni-performance-multi-replicas + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_3 - - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" - timeout_in_minutes: 45 - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model" - mirror_hardwares: h100_4 + # - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" + # timeout_in_minutes: 45 + # commands: + # - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model" + # mirror_hardwares: h100_4 - - group: ":card_index_dividers: TTS Model Test" - key: nightly-tts-test-group - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" - steps: + # - group: ":card_index_dividers: TTS Model Test" + # key: nightly-tts-test-group + # depends_on: upload-nightly-pipeline + # if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" + # steps: - - label: ":full_moon: TTS · Function Test with L4" - timeout_in_minutes: 120 - commands: - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - export VLLM_USE_DEEP_GEMM="0" - - export VLLM_MOE_USE_DEEP_GEMM="0" - - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py - mirror_hardwares: l4_1 + # - label: ":full_moon: TTS · Function Test with L4" + # timeout_in_minutes: 120 + # commands: + # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + # - export VLLM_USE_DEEP_GEMM="0" + # - export VLLM_MOE_USE_DEEP_GEMM="0" + # - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py + # mirror_hardwares: l4_1 - - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' - " - mirror_hardwares: l4_1 + # - label: ":full_moon: TTS · Accuracy Test · VoxCPM2" + # timeout_in_minutes: 30 + # commands: + # - | + # timeout 30m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + # pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model' + # " + # mirror_hardwares: l4_1 - - label: ":full_moon: TTS · Perf Test" - key: nightly-tts-performance - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_2 + # - label: ":full_moon: TTS · Perf Test" + # key: nightly-tts-performance + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_2 - - label: ":full_moon: TTS · VoxCPM2 · Perf Test" - key: nightly-tts-performance-voxcpm2 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 + # - label: ":full_moon: TTS · VoxCPM2 · Perf Test" + # key: nightly-tts-performance-voxcpm2 + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_1 - - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" - key: nightly-tts-performance-higgs-audio-v3 - timeout_in_minutes: 180 - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_USE_DEEP_GEMM="0" - - export VLLM_MOE_USE_DEEP_GEMM="0" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT - mirror_hardwares: h100_1 + # - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test" + # key: nightly-tts-performance-higgs-audio-v3 + # timeout_in_minutes: 180 + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - export VLLM_USE_DEEP_GEMM="0" + # - export VLLM_MOE_USE_DEEP_GEMM="0" + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT + # mirror_hardwares: h100_1 - - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" - key: nightly-tiny-model-test-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" - steps: + # - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" + # key: nightly-tiny-model-test-group + # depends_on: upload-nightly-pipeline + # if: >- + # build.env("NIGHTLY") == "1" || + # build.pull_request.labels includes "nightly-test" + # steps: - - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)" - timeout_in_minutes: 30 - # NOTE: Single-GPU (core_model) tiny model tests run on the ready label, - # so nightly only adds the multi-GPU parallelism configurations. - # - # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and - # --run-level `core_model` is used to indicate that we should use tiny model weights - # for the CI, since these tests should pass irrespective of whether or not we use - # the full weights or not. - commands: - - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model" - mirror_hardwares: l4_4 + # - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)" + # timeout_in_minutes: 30 + # # NOTE: Single-GPU (core_model) tiny model tests run on the ready label, + # # so nightly only adds the multi-GPU parallelism configurations. + # # + # # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and + # # --run-level `core_model` is used to indicate that we should use tiny model weights + # # for the CI, since these tests should pass irrespective of whether or not we use + # # the full weights or not. + # commands: + # - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_4 - - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" - key: nightly-diffusion-x2iat-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2iat-test" - steps: - # Use explicit file shards so single-card cases do not reserve 2+ GPUs, - # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. + # - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" + # key: nightly-diffusion-x2iat-group + # depends_on: upload-nightly-pipeline + # if: >- + # build.env("NIGHTLY") == "1" || + # build.pull_request.labels includes "nightly-test" || + # build.pull_request.labels includes "diffusion-x2iat-test" + # steps: + # # Use explicit file shards so single-card cases do not reserve 2+ GPUs, + # # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep. - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - tests/e2e/online_serving/test_boogu_image.py - tests/e2e/online_serving/test_boogu_image_edit.py - tests/e2e/online_serving/test_boogu_image_expansion.py - tests/e2e/online_serving/test_boogu_image_edit_expansion.py - tests/e2e/online_serving/test_bagel_expansion.py - tests/e2e/online_serving/test_krea2_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py - -m "full_model and diffusion and H100 and not distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" + # timeout_in_minutes: 120 + # commands: + # - >- + # pytest -sv + # tests/e2e/online_serving/test_qwen_image_expansion.py + # tests/e2e/online_serving/test_qwen_image_edit_expansion.py + # tests/e2e/online_serving/test_qwen_image_layered_expansion.py + # tests/e2e/online_serving/test_boogu_image.py + # tests/e2e/online_serving/test_boogu_image_edit.py + # tests/e2e/online_serving/test_boogu_image_expansion.py + # tests/e2e/online_serving/test_boogu_image_edit_expansion.py + # tests/e2e/online_serving/test_bagel_expansion.py + # tests/e2e/online_serving/test_krea2_expansion.py + # tests/e2e/offline_inference/test_krea2_expansion.py + # -m "full_model and diffusion and H100 and not distributed_cuda" + # --run-level "full_model" + # mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_qwen_image_expansion.py - tests/e2e/online_serving/test_qwen_image_edit_expansion.py - tests/e2e/offline_inference/test_krea2_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image" + # timeout_in_minutes: 120 + # commands: + # - >- + # pytest -sv + # tests/e2e/online_serving/test_qwen_image_expansion.py + # tests/e2e/online_serving/test_qwen_image_edit_expansion.py + # tests/e2e/offline_inference/test_krea2_expansion.py + # -m "full_model and diffusion and H100 and distributed_cuda" + # --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" - timeout_in_minutes: 90 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_qwen_image_layered_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX" + # timeout_in_minutes: 90 + # commands: + # - >- + # pytest -sv + # tests/e2e/online_serving/test_qwen_image_layered_expansion.py + # -m "full_model and diffusion and H100 and distributed_cuda" + # --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_bagel_expansion.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel" + # timeout_in_minutes: 120 + # commands: + # - >- + # pytest -sv + # tests/e2e/online_serving/test_bagel_expansion.py + # -m "full_model and diffusion and H100 and distributed_cuda" + # --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" - timeout_in_minutes: 120 - commands: - - >- - pytest -sv - tests/e2e/online_serving/test_glm_image_expansion.py - tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py - -m "full_model and diffusion and H100" - --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100" + # timeout_in_minutes: 120 + # commands: + # - >- + # pytest -sv + # tests/e2e/online_serving/test_glm_image_expansion.py + # tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py + # -m "full_model and diffusion and H100" + # --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" - timeout_in_minutes: 120 - commands: - - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy - mirror_hardwares: l4_4 + # - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" + # timeout_in_minutes: 120 + # commands: + # - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy + # mirror_hardwares: l4_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" + # timeout_in_minutes: 60 + # commands: + # - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" - timeout_in_minutes: 180 - commands: - - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 - - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" + # timeout_in_minutes: 180 + # commands: + # - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 + # - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model + # - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model + # mirror_hardwares: h100_1 - - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" - timeout_in_minutes: 180 - commands: - - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct" - - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3" - - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model - mirror_hardwares: h100_4 + # - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" + # timeout_in_minutes: 180 + # commands: + # - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct" + # - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3" + # - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model + # mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image" - key: nightly-diffusion-x2iat-performance-qwen-image - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.3.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 + # - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image" + # key: nightly-diffusion-x2iat-performance-qwen-image + # timeout_in_minutes: 180 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - export CACHE_DIT_VERSION=1.3.0 + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # exit $$EXIT + # mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" - key: nightly-diffusion-x2iat-performance-qwen-image-edit - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.3.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 + # - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit" + # key: nightly-diffusion-x2iat-performance-qwen-image-edit + # timeout_in_minutes: 180 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - export CACHE_DIT_VERSION=1.3.0 + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # exit $$EXIT + # mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.3.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_4 + # - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" + # key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 + # timeout_in_minutes: 180 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - export CACHE_DIT_VERSION=1.3.0 + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # exit $$EXIT + # mirror_hardwares: h100_4 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" - key: nightly-diffusion-x2iat-performance-qwen-image-layered - timeout_in_minutes: 120 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.3.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered" + # key: nightly-diffusion-x2iat-performance-qwen-image-layered + # timeout_in_minutes: 120 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - export CACHE_DIT_VERSION=1.3.0 + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # exit $$EXIT + # mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" - key: nightly-diffusion-x2iat-performance-bagel - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - export CACHE_DIT_VERSION=1.3.0 - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - exit $$EXIT - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL" + # key: nightly-diffusion-x2iat-performance-bagel + # timeout_in_minutes: 180 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - export CACHE_DIT_VERSION=1.3.0 + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # exit $$EXIT + # mirror_hardwares: h100_1 - - group: ":card_index_dividers: Diffusion X2V Model Test" - key: nightly-diffusion-x2v-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2v-test" - steps: + # - group: ":card_index_dividers: Diffusion X2V Model Test" + # key: nightly-diffusion-x2v-group + # depends_on: upload-nightly-pipeline + # if: >- + # build.env("NIGHTLY") == "1" || + # build.pull_request.labels includes "nightly-test" || + # build.pull_request.labels includes "diffusion-x2v-test" + # steps: - - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test" + # timeout_in_minutes: 90 + # commands: + # - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test" + # timeout_in_minutes: 90 + # commands: + # - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" + # timeout_in_minutes: 90 + # commands: + # - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model" + # mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Doc Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model" - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion X2V · Doc Test" + # timeout_in_minutes: 60 + # commands: + # - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model" + # mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2V · Other Function Test" - timeout_in_minutes: 90 - commands: - - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" - - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Other Function Test" + # timeout_in_minutes: 90 + # commands: + # - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model" + # - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" + # timeout_in_minutes: 180 + # commands: + # - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model + # - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · Other Accuracy Test" - timeout_in_minutes: 180 - commands: - - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model - - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Other Accuracy Test" + # timeout_in_minutes: 180 + # commands: + # - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model + # - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model + # - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model + # - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model + # mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · Perf Test" - key: nightly-diffusion-x2v-performance - timeout_in_minutes: 180 - commands: - - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json - EXIT1=$$? - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json - EXIT2=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi - exit $$EXIT2 - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion X2V · Perf Test" + # key: nightly-diffusion-x2v-performance + # timeout_in_minutes: 180 + # commands: + # - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + # - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json + # EXIT1=$$? + # pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json + # EXIT2=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + # buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + # if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi + # exit $$EXIT2 + # mirror_hardwares: h100_2 - - group: ":card_index_dividers: Diffusion Test" - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" - steps: + # - group: ":card_index_dividers: Diffusion Test" + # depends_on: upload-nightly-pipeline + # if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test" + # steps: - - label: ":full_moon: Diffusion · Offloader Test" - timeout_in_minutes: 40 - commands: - - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda" - mirror_hardwares: l4_1 + # - label: ":full_moon: Diffusion · Offloader Test" + # timeout_in_minutes: 40 + # commands: + # - timeout 40m pytest -s -v tests/diffusion/offloader -m "full_model and cuda" + # mirror_hardwares: l4_1 - - label: ":full_moon: Diffusion · Quantization Test with H100" - commands: - - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" - mirror_hardwares: h100_1 + # - label: ":full_moon: Diffusion · Quantization Test with H100" + # commands: + # - timeout 60m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model" + # mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion · Quantization Test with L4" - timeout_in_minutes: 60 - commands: - - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" - mirror_hardwares: l4_1 + # - label: ":full_moon: Diffusion · Quantization Test with L4" + # timeout_in_minutes: 60 + # commands: + # - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model" + # mirror_hardwares: l4_1 - - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test" - timeout_in_minutes: 60 - commands: - - >- - pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py - -m "full_model and diffusion and H100 and distributed_cuda" - --run-level "full_model" - mirror_hardwares: h100_2 + # - label: ":full_moon: Diffusion · Wan VAE Spatial Shard Correctness Test" + # timeout_in_minutes: 60 + # commands: + # - >- + # pytest -sv tests/diffusion/distributed/test_wan_spatial_shard.py + # -m "full_model and diffusion and H100 and distributed_cuda" + # --run-level "full_model" + # mirror_hardwares: h100_2 - - label: ":bar_chart: Testcase Statistics" - key: nightly-testcase-statistics - timeout_in_minutes: 120 - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" - commands: - - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html - - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" - mirror_hardwares: h100_1 + # - label: ":bar_chart: Testcase Statistics" + # key: nightly-testcase-statistics + # timeout_in_minutes: 120 + # depends_on: upload-nightly-pipeline + # if: build.env("NIGHTLY") == "1" + # commands: + # - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html + # - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" + # mirror_hardwares: h100_1 - # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. - # Do not add new depends_on entries or extend this step. - - label: ":email: Nightly Collection & Email" - key: nightly-perf-distribution - depends_on: - - nightly-omni-performance-no-async-chunk - - nightly-omni-performance-async-chunk - - nightly-omni-performance-vllm-text - - nightly-omni-performance-multi-replicas - - nightly-tts-performance - - nightly-tts-performance-higgs-audio-v3 - - nightly-diffusion-x2iat-performance-qwen-image - - nightly-diffusion-x2iat-performance-qwen-image-edit - - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - nightly-diffusion-x2iat-performance-qwen-image-layered - - nightly-diffusion-x2iat-performance-bagel - - nightly-diffusion-x2v-performance - - nightly-testcase-statistics - if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" - commands: - - pip install openpyxl - - export DEFAULT_INPUT_DIR=tests/dfx/perf/results - - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3 - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - - python tools/nightly/generate_nightly_perf_excel.py - - python tools/nightly/generate_nightly_perf_html.py - - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" - - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx" - - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" - agents: - queue: "cpu_queue_premerge" + # # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. + # # Do not add new depends_on entries or extend this step. + # - label: ":email: Nightly Collection & Email" + # key: nightly-perf-distribution + # depends_on: + # - nightly-omni-performance-no-async-chunk + # - nightly-omni-performance-async-chunk + # - nightly-omni-performance-vllm-text + # - nightly-omni-performance-multi-replicas + # - nightly-tts-performance + # - nightly-tts-performance-higgs-audio-v3 + # - nightly-diffusion-x2iat-performance-qwen-image + # - nightly-diffusion-x2iat-performance-qwen-image-edit + # - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 + # - nightly-diffusion-x2iat-performance-qwen-image-layered + # - nightly-diffusion-x2iat-performance-bagel + # - nightly-diffusion-x2v-performance + # - nightly-testcase-statistics + # if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" + # commands: + # - pip install openpyxl + # - export DEFAULT_INPUT_DIR=tests/dfx/perf/results + # - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3 + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel + # - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance + # - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics + # - python tools/nightly/generate_nightly_perf_excel.py + # - python tools/nightly/generate_nightly_perf_html.py + # - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" + # - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx" + # - buildkite-agent artifact upload "tests/dfx/perf/results/*.html" + # agents: + # queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index b0ea85900..a05cfa3ca 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -18,218 +18,139 @@ steps: - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" mirror_hardwares: l4_1 - - label: "Custom Pipeline Test" - depends_on: upload-ready-pipeline - commands: - - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" - mirror_hardwares: l4_1 + # - label: "Custom Pipeline Test" + # depends_on: upload-ready-pipeline + # commands: + # - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" + # mirror_hardwares: l4_1 - - group: ":card_index_dividers: Diffusion Test" - depends_on: upload-ready-pipeline - steps: + # - group: ":card_index_dividers: Diffusion Test" + # depends_on: upload-ready-pipeline + # steps: - - label: "Diffusion · Offloader Test" - commands: - - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Diffusion · Offloader Test" + # commands: + # - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" + # mirror_hardwares: l4_1 - - label: "Diffusion · Cache Test" - commands: - - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Diffusion · Cache Test" + # commands: + # - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_1 - - label: "Diffusion · Batch Test" - commands: - - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Diffusion · Batch Test" + # commands: + # - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_1 - - label: "Diffusion · Distributed Test" - source_file_dependencies: - - tests/diffusion/distributed/ - - vllm_omni/diffusion/distributed/ - - vllm_omni/diffusion/attention/parallel/ - - vllm_omni/diffusion/data.py - - vllm_omni/diffusion/models/ - commands: - - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_4 + # - label: "Diffusion · Distributed Test" + # source_file_dependencies: + # - tests/diffusion/distributed/ + # - vllm_omni/diffusion/distributed/ + # - vllm_omni/diffusion/attention/parallel/ + # - vllm_omni/diffusion/data.py + # - vllm_omni/diffusion/models/ + # commands: + # - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_4 - - label: "Diffusion · Model Test" - commands: - - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Diffusion · Model Test" + # commands: + # - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_1 - - label: "Diffusion · Other Test" - commands: - - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Diffusion · Other Test" + # commands: + # - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_1 - - group: ":card_index_dividers: Tiny Model Tests [Single GPU]" - depends_on: upload-ready-pipeline - steps: + # - group: ":card_index_dividers: Tiny Model Tests [Single GPU]" + # depends_on: upload-ready-pipeline + # steps: - - label: "Tiny Model Tests · Diffusion (base)" - commands: - - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" - mirror_hardwares: l4_1 + # - label: "Tiny Model Tests · Diffusion (base)" + # commands: + # - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" + # mirror_hardwares: l4_1 - - label: "Engine&Model Executor Test" - depends_on: upload-ready-pipeline - commands: - - | - timeout 15m bash -c " - pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "Engine&Model Executor Test" + # depends_on: upload-ready-pipeline + # commands: + # - | + # timeout 15m bash -c " + # pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "Distributed Test" - depends_on: upload-ready-pipeline - commands: - - | - timeout 20m bash -c " - pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "Distributed Test" + # depends_on: upload-ready-pipeline + # commands: + # - | + # timeout 20m bash -c " + # pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "Entrypoints Test" - depends_on: upload-ready-pipeline - commands: - - | - timeout 30m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "Entrypoints Test" + # depends_on: upload-ready-pipeline + # commands: + # - | + # timeout 30m bash -c " + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - group: ":card_index_dividers: E2E Test" depends_on: upload-ready-pipeline steps: - - label: "RLHF · VeRL-Omni E2E Test" - source_file_dependencies: - - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py - - tests/e2e/offline_inference/custom_pipeline/ - - vllm_omni/diffusion/ - - vllm_omni/engine/ - - vllm_omni/entrypoints/ - - vllm_omni/inputs/ - - vllm_omni/outputs/ - commands: - - | - timeout 20m bash -c " - pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py - " - mirror_hardwares: l4_1 + # - label: "RLHF · VeRL-Omni E2E Test" + # source_file_dependencies: + # - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py + # - tests/e2e/offline_inference/custom_pipeline/ + # - vllm_omni/diffusion/ + # - vllm_omni/engine/ + # - vllm_omni/entrypoints/ + # - vllm_omni/inputs/ + # - vllm_omni/outputs/ + # commands: + # - | + # timeout 20m bash -c " + # pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py + # " + # mirror_hardwares: l4_1 - - label: "Omni · Qwen3-Omni Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_omni.py - - vllm_omni/model_executor/models/qwen3_omni/ - - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py - - vllm_omni/deploy/qwen3_omni_moe.yaml - commands: - - | - timeout 20m bash -c " - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_2 + # - label: "Omni · Qwen3-Omni Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen3_omni.py + # - vllm_omni/model_executor/models/qwen3_omni/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py + # - vllm_omni/deploy/qwen3_omni_moe.yaml + # commands: + # - | + # timeout 20m bash -c " + # pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Offline Test" - source_file_dependencies: - - tests/e2e/offline_inference/test_minicpmo_4_5.py - - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | timeout 20m bash -c " pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate + mirror_hardwares: h100_2 - label: "Omni · MiniCPM-o 4.5 Online Test" - source_file_dependencies: - - tests/e2e/online_serving/test_minicpmo_4_5.py - - vllm_omni/deploy/minicpmo_4_5.yaml commands: - | timeout 20m bash -c " pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' " - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 2 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - + mirror_hardwares: h100_2 + - label: "Omni · MiniCPM-o 4.5 Duplex Test" - source_file_dependencies: - - tests/assets/minicpmo_4_5/response_required_16k.wav - - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav - - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py - - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py - - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py - - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py - - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py - - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py - - tests/helpers/minicpmo_4_5_duplex.py - - vllm_omni/deploy/minicpmo_4_5_duplex.yaml - - vllm_omni/experimental/fullduplex/ - - vllm_omni/model_executor/models/minicpmo_4_5/ - - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py - - vllm_omni/core/sched/ - - vllm_omni/outputs/multimodal_accumulation.py - - vllm_omni/engine/stage_init_utils.py - - vllm_omni/engine/async_omni_engine.py - - vllm_omni/entrypoints/cli/serve.py commands: - | timeout 20m bash -c " @@ -237,147 +158,147 @@ steps: " mirror_hardwares: h100_2 - - label: "TTS · VoxCPM2 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_voxcpm2_tts.py - - vllm_omni/model_executor/models/voxcpm2/ - - vllm_omni/deploy/voxcpm2.yaml - timeout_in_minutes: 30 - commands: - - | - timeout 30m bash -c " - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · VoxCPM2 Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_voxcpm2_tts.py + # - vllm_omni/model_executor/models/voxcpm2/ + # - vllm_omni/deploy/voxcpm2.yaml + # timeout_in_minutes: 30 + # commands: + # - | + # timeout 30m bash -c " + # export VLLM_LOGGING_LEVEL=DEBUG + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · Qwen3-TTS CustomVoice Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_customvoice.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · Qwen3-TTS CustomVoice Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen3_tts_customvoice.py + # - vllm_omni/model_executor/models/qwen3_tts/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py + # - vllm_omni/deploy/qwen3_tts.yaml + # commands: + # - | + # timeout 20m bash -c " + # export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + # pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · Qwen3-TTS Base Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen3_tts_base.py - - vllm_omni/model_executor/models/qwen3_tts/ - - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py - - vllm_omni/deploy/qwen3_tts.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 - export VLLM_OMNI_USE_V2_RUNNER=1 - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · Qwen3-TTS Base Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen3_tts_base.py + # - vllm_omni/model_executor/models/qwen3_tts/ + # - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py + # - vllm_omni/deploy/qwen3_tts.yaml + # commands: + # - | + # timeout 20m bash -c " + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + # export VLLM_OMNI_USE_V2_RUNNER=1 + # pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "TTS · Higgs-Audio-V3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_higgs_audio_v3.py - - vllm_omni/model_executor/models/higgs_audio_v3/ - - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py - - vllm_omni/deploy/higgs_multimodal_qwen3.yaml - commands: - - | - timeout 20m bash -c " - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_USE_DEEP_GEMM=0 - export VLLM_MOE_USE_DEEP_GEMM=0 - pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: l4_1 + # - label: "TTS · Higgs-Audio-V3 Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_higgs_audio_v3.py + # - vllm_omni/model_executor/models/higgs_audio_v3/ + # - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py + # - vllm_omni/deploy/higgs_multimodal_qwen3.yaml + # commands: + # - | + # timeout 20m bash -c " + # export VLLM_WORKER_MULTIPROC_METHOD=spawn + # export VLLM_USE_DEEP_GEMM=0 + # export VLLM_MOE_USE_DEEP_GEMM=0 + # pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: l4_1 - - label: "Diffusion · Qwen Image Edit Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_edit.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 20m bash -c " - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Edit Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image_edit.py + # - vllm_omni/diffusion/models/qwen_image/ + # commands: + # - | + # timeout 20m bash -c " + # pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Bagel Test" - source_file_dependencies: - - tests/e2e/online_serving/test_bagel.py - - vllm_omni/model_executor/models/bagel/ - - vllm_omni/diffusion/models/bagel/ - - vllm_omni/model_executor/stage_input_processors/bagel.py - - vllm_omni/deploy/bagel.yaml - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Bagel Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_bagel.py + # - vllm_omni/model_executor/models/bagel/ + # - vllm_omni/diffusion/models/bagel/ + # - vllm_omni/model_executor/stage_input_processors/bagel.py + # - vllm_omni/deploy/bagel.yaml + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Wan22 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_wan22_t2v.py - - vllm_omni/diffusion/models/wan2_2/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Wan22 Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_wan22_t2v.py + # - vllm_omni/diffusion/models/wan2_2/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Cosmos3 Test" - source_file_dependencies: - - tests/e2e/online_serving/test_cosmos3.py - - vllm_omni/diffusion/models/cosmos3/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Cosmos3 Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_cosmos3.py + # - vllm_omni/diffusion/models/cosmos3/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image.py + # - vllm_omni/diffusion/models/qwen_image/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · Qwen Image Layered Test" - source_file_dependencies: - - tests/e2e/online_serving/test_qwen_image_layered.py - - vllm_omni/diffusion/models/qwen_image/ - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' - " - mirror_hardwares: h100_1 + # - label: "Diffusion · Qwen Image Layered Test" + # source_file_dependencies: + # - tests/e2e/online_serving/test_qwen_image_layered.py + # - vllm_omni/diffusion/models/qwen_image/ + # commands: + # - | + # timeout 40m bash -c " + # export VLLM_IMAGE_FETCH_TIMEOUT=60 + # pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' + # " + # mirror_hardwares: h100_1 - - label: "Diffusion · GGUF Plugin Test" - soft_fail: true - source_file_dependencies: - - tests/plugin/test_gguf_diffusion.py - - vllm_omni/diffusion/model_loader - commands: - - pip install "vllm-gguf-plugin>=0.0.3" - - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py - mirror_hardwares: l4_1 + # - label: "Diffusion · GGUF Plugin Test" + # soft_fail: true + # source_file_dependencies: + # - tests/plugin/test_gguf_diffusion.py + # - vllm_omni/diffusion/model_loader + # commands: + # - pip install "vllm-gguf-plugin>=0.0.3" + # - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py + # mirror_hardwares: l4_1 diff --git a/.buildkite/intel/scripts/bootstrap-intel-omni.sh b/.buildkite/intel/scripts/bootstrap-intel-omni.sh index a5d3f00a6..9038fe568 100755 --- a/.buildkite/intel/scripts/bootstrap-intel-omni.sh +++ b/.buildkite/intel/scripts/bootstrap-intel-omni.sh @@ -4,6 +4,10 @@ set -euo pipefail +# minicpm-challenge branch: skip entire Intel/XPU CI (no pipeline upload => no image build / tests). +echo "--- Skipping Intel/XPU CI on this branch (including image build)" +exit 0 + source .buildkite/common/scripts/resolve_skip_ci.sh upload_pipeline() { diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index e05178c52..36b9c4055 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -1,39 +1,39 @@ steps: - - group: ":card_index_dividers: Diffusion X2V Model Test" - key: nightly-diffusion-x2v-group - depends_on: upload-nightly-pipeline - if: >- - build.env("NIGHTLY") == "1" || - build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2v-test" - steps: - - label: ":full_moon: Diffusion X2V · Function Test" - mirror_hardwares: a2b3_npu_4 - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - HF_TOKEN: "${HF_TOKEN}" - commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" - - group: ":card_index_dividers: TTS Model Test" - key: nightly-tts-test-group - depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" - steps: - - label: ":full_moon: TTS · Perf Test" - mirror_hardwares: a3_npu_2 - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" - HF_TOKEN: "${HF_TOKEN}" - commands: - - export BENCHMARK_DIR=tests/dfx/perf/results - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - | - set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json - EXIT=$$? - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" - exit $$EXIT + # - group: ":card_index_dividers: Diffusion X2V Model Test" + # key: nightly-diffusion-x2v-group + # depends_on: upload-nightly-pipeline + # if: >- + # build.env("NIGHTLY") == "1" || + # build.pull_request.labels includes "nightly-test" || + # build.pull_request.labels includes "diffusion-x2v-test" + # steps: + # - label: ":full_moon: Diffusion X2V · Function Test" + # mirror_hardwares: a2b3_npu_4 + # env: + # VLLM_WORKER_MULTIPROC_METHOD: spawn + # HF_TOKEN: "${HF_TOKEN}" + # commands: + # - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" + # - group: ":card_index_dividers: TTS Model Test" + # key: nightly-tts-test-group + # depends_on: upload-nightly-pipeline + # if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" + # steps: + # - label: ":full_moon: TTS · Perf Test" + # mirror_hardwares: a3_npu_2 + # env: + # VLLM_WORKER_MULTIPROC_METHOD: spawn + # VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + # HF_TOKEN: "${HF_TOKEN}" + # commands: + # - export BENCHMARK_DIR=tests/dfx/perf/results + # - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + # - | + # set +e + # pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json + # EXIT=$$? + # buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + # exit $$EXIT - group: ":card_index_dividers: Omni Model Test" key: nightly-omni-group @@ -41,15 +41,7 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - label: ":robot: Omni · MiniCPM-o 4.5 Test" - agents: - queue: "ascend-a3" - resource_class: "npu-2" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index 956e8acd6..e27fcfa1f 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -4,15 +4,7 @@ steps: depends_on: upload-ready-pipeline steps: - label: ":robot: Omni · MiniCPM-o 4.5 Test" - agents: - queue: "ascend-a3" - resource_class: "npu-2" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" @@ -20,15 +12,15 @@ steps: commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' - - group: ":card_index_dividers: TTS Model Test" - key: ready-tts-group - depends_on: upload-ready-pipeline - steps: - - label: ":sound: TTS · Qwen3-TTS CustomVoice Test" - mirror_hardwares: a2b3_npu_1 - env: - VLLM_WORKER_MULTIPROC_METHOD: spawn - VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" - HF_TOKEN: "${HF_TOKEN}" - commands: - - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' + # - group: ":card_index_dividers: TTS Model Test" + # key: ready-tts-group + # depends_on: upload-ready-pipeline + # steps: + # - label: ":sound: TTS · Qwen3-TTS CustomVoice Test" + # mirror_hardwares: a2b3_npu_1 + # env: + # VLLM_WORKER_MULTIPROC_METHOD: spawn + # VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + # HF_TOKEN: "${HF_TOKEN}" + # commands: + # - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index a8855e04b..3d5c42517 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -31,7 +31,6 @@ def get_question(prompt_type: str = "text") -> str: return prompts.get(prompt_type, prompts["text"]) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni @@ -88,7 +87,6 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index a4fefec98..f1b5ae178 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -60,7 +60,6 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni @@ -197,7 +196,6 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni