vllm-omni/.buildkite/cuda/test-merge.yml

# CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main
# (both diff-aware via source_file_dependencies).

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Simple · Model Executor Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/model_executor -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Diffusion Test"
        timeout_in_minutes: 30
        commands:
          - "pytest -sv tests/diffusion -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Engine&Entrypoints Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Other Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: Diffusion Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Diffusion · LoRA Test"
        timeout_in_minutes: 30
        commands:
          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
        mirror_hardwares: l4_1

  - label: "Distributed Test"
    depends_on: upload-merge-pipeline
    timeout_in_minutes: 60
    commands:
      - |
        timeout 55m bash -c "
          set -e
          export VLLM_IMAGE_FETCH_TIMEOUT=60
          pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
        "
    mirror_hardwares: h100_1

  - group: ":card_index_dividers: Entrypoints Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Entrypoint Test with H100"
        timeout_in_minutes: 60
        commands:
          # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
          # (see the P0 thread on #7413); the test stays in the tree and runs again there.
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit
        mirror_hardwares: h100_2

      - label: "Entrypoint Test with L4"
        timeout_in_minutes: 60
        # Workaround https://github.com/vllm-project/vllm-omni/issues/8016: shutdown()
        # leaves inline UniProc DiffusionWorker VRAM mapped in the pytest parent.
        # This job does not collect the ready leftover
        # (test_diffusion_generate_request_id, core_model); isolate the
        # advanced_model DiT handshake instead. Revert to a single sweep after
        # the product shutdown fix.
        commands:
          - |
            set -eu
            pytest -s -v tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model"
            pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model" --deselect tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake
        mirror_hardwares: l4_1

      - label: "Entrypoint Test with L4 · 2-GPU"
        timeout_in_minutes: 60
        commands:
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_2" --run-level "advanced_model"
        mirror_hardwares: l4_2

  - group: ":card_index_dividers: E2E Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "TTS · Qwen3-TTS CustomVoice Test"
        source_file_dependencies: tts_qwen3_tts_function
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        mirror_hardwares: l4_1

      - label: "TTS · Qwen3-TTS Base Test"
        source_file_dependencies: tts_qwen3_tts_cov
        # 40 -> 60: the coverage split runs the model twice.
        timeout_in_minutes: 60
        commands:
          - |
            export VLLM_LOGGING_LEVEL=DEBUG
            export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
            .buildkite/common/scripts/run_cov_split.sh \
              --model-id qwen3_tts \
              --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
              --online tests/e2e/online_serving/test_qwen3_tts_base.py \
              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: l4_1

      - label: "Omni · Qwen3-Omni Test"
        source_file_dependencies: omni_qwen3_omni_function
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_2

      - label: "Omni · MiniCPM-o 4.5 Test"
        source_file_dependencies: omni_minicpmo_4_5_function
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        # h100_2, not h100_1: advanced_model asserts transcribe every audio
        # response with Whisper, and tests/helpers/media.py requires at least
        # 16 GiB free VRAM on a device to run Whisper on GPU. When the model
        # server leaves insufficient memory, Whisper falls back to CPU:
        # ~8-10 min per clip vs ~30 s, which overruns the 50-minute job timeout
        # (build 2956). h100_2 guarantees a second GPU with ample free VRAM.
        mirror_hardwares: h100_2

      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
        source_file_dependencies: omni_minicpmo_4_5_duplex_function
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_minicpmo_4_5_window.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_1

      - label: "Diffusion · Qwen Image Test"
        source_file_dependencies: diffusion_qwen_image_function
        commands:
          - |
            timeout 40m bash -c '
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            '
        mirror_hardwares: h100_1

      - label: "Diffusion · Wan22 Test"
        source_file_dependencies: diffusion_wan22_function
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1

      - label: "Diffusion · HunyuanVideo-1.5 Test"
        source_file_dependencies: diffusion_hunyuan_video_function
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1

      - label: "Diffusion · LingBot-Video Test"
        source_file_dependencies: diffusion_lingbot_function
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1

      - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
        source_file_dependencies: diffusion_minimax_h3_function
        # Keep four server lifecycles in one Buildkite job, but never in one
        # pytest process. DLO uses multiprocessing shared-memory queues; the
        # parent's resource_tracker can retain a stale /psm_* entry during
        # teardown and strand the next Ref2VA collective.
        # A per-file process plus a hard timeout also makes a regression fail
        # instead of leaving the merge queue stuck indefinitely.
        timeout_in_minutes: 150
        commands:
          - |
            set -eu
            export VLLM_IMAGE_FETCH_TIMEOUT=60
            if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
              export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
            fi
            if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
              export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
            fi
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
        mirror_hardwares: h100_2