vllm-omni/.buildkite/cuda/test-merge.yml

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  VLLM_USE_FLASHINFER_MOE_FP16: 0
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  - group: ":card_index_dividers: Simple Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Simple · Model Executor Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/model_executor -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Diffusion Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/diffusion -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Engine&Entrypoints Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
        mirror_hardwares: l4_1

      - label: "Simple · Other Test"
        timeout_in_minutes: 20
        commands:
          - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: Diffusion Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Diffusion · LoRA Test"
        timeout_in_minutes: 30
        commands:
          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
        mirror_hardwares: l4_1

  - label: "Distributed Test"
    depends_on: upload-merge-pipeline
    timeout_in_minutes: 60
    commands:
      - |
        timeout 55m bash -c "
          set -e
          export VLLM_IMAGE_FETCH_TIMEOUT=60
          pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
        "
    mirror_hardwares: h100_1

  - group: ":card_index_dividers: Entrypoints Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "Entrypoint Test with H100"
        # TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
        # run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
        # 2915 vs 2916), which pushed this suite past 60 min. Revert once the
        # 0.26 startup regression is understood and fixed.
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
        mirror_hardwares: h100_2

      - label: "Entrypoint Test with L4"
        timeout_in_minutes: 60
        commands:
          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
        mirror_hardwares: l4_1

  - group: ":card_index_dividers: E2E Test"
    depends_on: upload-merge-pipeline
    steps:
      - label: "TTS · Qwen3-TTS CustomVoice Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
          - vllm_omni/model_executor/models/qwen3_tts/
          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
          - vllm_omni/model_executor/models/common/snake_activation.py
          - vllm_omni/model_executor/models/common/alias_free_activation.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_LOGGING_LEVEL=DEBUG
              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            "
        mirror_hardwares: l4_1

      - label: "TTS · Qwen3-TTS Base Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_qwen3_tts_base.py
          - tests/e2e/offline_inference/test_qwen3_tts_base.py
          - vllm_omni/model_executor/models/qwen3_tts/
          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
          - vllm_omni/model_executor/models/common/snake_activation.py
          - vllm_omni/model_executor/models/common/alias_free_activation.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
          - .buildkite/common/scripts/run_cov_split.sh
          - pyproject.toml
        # 40 -> 60: the coverage split runs the model twice.
        timeout_in_minutes: 60
        commands:
          - |
            export VLLM_LOGGING_LEVEL=DEBUG
            export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
            .buildkite/common/scripts/run_cov_split.sh \
              --model-id qwen3_tts \
              --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
              --online tests/e2e/online_serving/test_qwen3_tts_base.py \
              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: l4_1

      - label: "Omni · Qwen3-Omni Test"
        source_file_dependencies:
          - tests/e2e/offline_inference/test_qwen3_omni.py
          - tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
          - tests/e2e/online_serving/test_qwen3_omni.py
          - tests/helpers/runtime.py
          - vllm_omni/entrypoints/openai/serving_chat.py
          - vllm_omni/model_executor/models/qwen3_omni/
          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
          - vllm_omni/model_executor/models/common/snake_activation.py
          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
          - vllm_omni/deploy/qwen3_omni_moe.yaml
          - vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_2

      - label: "Omni · MiniCPM-o 4.5 Test"
        source_file_dependencies:
          - tests/e2e/offline_inference/test_minicpmo_4_5.py
          - tests/e2e/online_serving/test_minicpmo_4_5.py
          - vllm_omni/model_executor/models/minicpmo_4_5/
          - vllm_omni/model_executor/models/step_audio2/
          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
          - vllm_omni/model_executor/models/cosyvoice3/utils.py
          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
          - vllm_omni/deploy/minicpmo_4_5.yaml
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        # h100_2, not h100_1: advanced_model asserts transcribe every audio
        # response with Whisper, and tests/helpers/media.py only uses an
        # accelerator for Whisper when a spare GPU exists. On a single card it
        # grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
        # 50-minute job timeout (build 2956).
        mirror_hardwares: h100_2

      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
        source_file_dependencies:
          - tests/assets/minicpmo_4_5/response_required_16k.wav
          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
          - vllm_omni/deploy/minicpmo_4_5.yaml
          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
          - vllm_omni/experimental/fullduplex/
          - vllm_omni/model_executor/models/minicpmo_4_5/
          - vllm_omni/model_executor/models/step_audio2/
          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
          - vllm_omni/model_executor/models/cosyvoice3/utils.py
          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
          - vllm_omni/core/sched/
          - vllm_omni/outputs/multimodal_accumulation.py
          - vllm_omni/engine/stage_init_utils.py
          - vllm_omni/engine/async_omni_engine.py
          - vllm_omni/engine/orchestrator.py
          - vllm_omni/entrypoints/cli/serve.py
          - vllm_omni/entrypoints/openai/api_server.py
          - vllm_omni/entrypoints/openai/duplex_capability.py
          - vllm_omni/config/stage_config.py
        timeout_in_minutes: 50
        commands:
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_1

      - label: "Diffusion · Qwen Image Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_qwen_image.py
          - vllm_omni/diffusion/models/qwen_image/
          - vllm_omni/diffusion/models/dmd2/
          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
        commands:
          - |
            timeout 40m bash -c '
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
            '
        mirror_hardwares: h100_1

      - label: "Diffusion · Wan22 Test"
        source_file_dependencies:
          - tests/e2e/offline_inference/test_wan22_t2v.py
          - tests/e2e/online_serving/test_wan22_t2v.py
          - vllm_omni/diffusion/models/wan2_2/
          - vllm_omni/diffusion/models/dmd2/
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1

      - label: "Diffusion · HunyuanVideo-1.5 Test"
        source_file_dependencies:
          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
          - vllm_omni/diffusion/models/hunyuan_video/
        commands:
          - |
            timeout 40m bash -c "
              export VLLM_IMAGE_FETCH_TIMEOUT=60
              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
            "
        mirror_hardwares: h100_1