vllm-omni/.buildkite/cuda/test-nightly.yml

# CUDA CI L4 (`full_model`): nightly and release share this file.
# Uploaded by Upload Nightly Pipeline: NIGHTLY=1, or PR labels (nightly-test / omni-test / ...).
# Hardware: omit mirror_hardwares to compose from -m SKU + cards_n; or set a preset string to force.

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
  - group: ":card_index_dividers: Omni Model Test"
    key: nightly-omni-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
        source_file_dependencies:
          - omni_qwen3_omni_function
          - omni_minicpmo_4_5_function
        timeout_in_minutes: 120
        commands:
          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py

      - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
        source_file_dependencies:
          - omni_qwen3_omni_function
          - omni_minicpmo_4_5_function
        timeout_in_minutes: 90
        commands:
          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py

      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
        source_file_dependencies: omni_minicpmo_4_5_duplex_function
        timeout_in_minutes: 120
        commands:
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"

      - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
        source_file_dependencies: omni_qwen2_5_omni_doc
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model"

      - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
        source_file_dependencies: omni_qwen3_omni_doc
        timeout_in_minutes: 90
        artifact_paths:
          - audio_*.wav
        commands:
          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model"

      - label: ":full_moon: Omni · Accuracy Test"
        source_file_dependencies: omni_qwen3_omni_accuracy
        timeout_in_minutes: 180
        artifact_paths:
          - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json
        commands:
          - export SEED_TTS_WER_EVAL=1
          - export SEED_TTS_EVAL_DEVICE=cuda:1
          - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model

      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
        source_file_dependencies: omni_minicpmo_4_5_accuracy
        timeout_in_minutes: 180
        artifact_paths:
          - tests/e2e/accuracy/minicpmo_4_5/results/*.json
        commands:
          - export SEED_TTS_WER_EVAL=1
          - export SEED_TTS_EVAL_DEVICE=cuda:1
          - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model

      - label: ":full_moon: Omni · Omni-DuplexEval CI Guard"
        source_file_dependencies: omni_duplex_eval_ci
        timeout_in_minutes: 240
        env:
          VLLM_DUPLEX_EVAL_JUDGE_URL: "http://127.0.0.1:8001"
        commands:
          - |
            set +e
            pytest -s -v tests/e2e/accuracy/omni_duplex_eval/test_omni_duplex_eval_ci.py \
              -m "full_model and H100 and B200 and cuda and cards_2" \
              --run-level full_model \
              --basetemp /tmp/omni_duplex_ci
            EXIT=$$?
            buildkite-agent artifact upload "tests/e2e/accuracy/omni_duplex_eval/results/*.json"
            exit $$EXIT

      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
        source_file_dependencies: omni_qwen3_omni_perf
        key: nightly-omni-performance-no-async-chunk
        timeout_in_minutes: 300
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2"

      - label: ":full_moon: Omni · Perf Test · Async Chunk"
        source_file_dependencies: omni_qwen3_omni_perf
        key: nightly-omni-performance-async-chunk
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2"

      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
        source_file_dependencies: omni_minicpmo_4_5_perf
        key: nightly-omni-performance-minicpmo-4-5
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1"

      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
        source_file_dependencies: omni_minicpmo_4_5_duplex_perf
        key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1"

      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
        source_file_dependencies: omni_qwen3_omni_function
        timeout_in_minutes: 45
        commands:
          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model"

  - group: ":card_index_dividers: Entrypoints Test"
    key: nightly-entrypoints-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: Entrypoint Test with H100"
        source_file_dependencies: entrypoints_function
        # Merge collects advanced_model (5 duplex interruptions). Nightly
        # collects full_model (20 interruptions + BAGEL sleep).
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/entrypoints/ -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"

  - group: ":card_index_dividers: TTS Model Test"
    key: nightly-tts-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: TTS · Function Test with L4"
        source_file_dependencies: tts_qwen3_tts_function
        timeout_in_minutes: 120
        env:
          VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR: .
        artifact_paths:
          - failed_speech_*.wav
          - failed_speech_*.json
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - export VLLM_USE_DEEP_GEMM="0"
          - export VLLM_MOE_USE_DEEP_GEMM="0"
          - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy

      - label: ":full_moon: TTS · Perf Test · Single-GPU"
        source_file_dependencies: tts_qwen3_tts_perf
        key: nightly-tts-performance-single
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1"

      - label: ":full_moon: TTS · Perf Test · 2-GPU"
        source_file_dependencies: tts_qwen3_tts_perf
        key: nightly-tts-performance-2gpu
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2"

  - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
    key: nightly-tiny-model-test-group
    depends_on: upload-nightly-pipeline
    if: >-
      build.env("NIGHTLY") == "1" ||
      build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
        source_file_dependencies: diffusion_tiny_model
        timeout_in_minutes: 30
        # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
        # so nightly only adds the multi-GPU parallelism configurations.
        #
        # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
        # --run-level `core_model` is used to indicate that we should use tiny model weights
        # for the CI, since these tests should pass irrespective of whether or not we use
        # the full weights or not.
        commands:
          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model"

      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
        source_file_dependencies: diffusion_tiny_model
        timeout_in_minutes: 30
        # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
        commands:
          - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model"

  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
    key: nightly-diffusion-x2iat-group
    depends_on: upload-nightly-pipeline
    if: >-
      build.env("NIGHTLY") == "1" ||
      build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
        source_file_dependencies: diffusion_qwen_image_function
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
            -m "full_model and diffusion and H100 and B200 and cards_1"
            --run-level "full_model"

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
        source_file_dependencies: diffusion_qwen_image_function
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
            -m "full_model and diffusion and H100 and B200 and cards_2"
            --run-level "full_model"

      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU"
        source_file_dependencies: diffusion_qwen_image_function
        timeout_in_minutes: 120
        commands:
          - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model"


      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
        source_file_dependencies: diffusion_text_to_image_doc
        timeout_in_minutes: 60
        commands:
          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model"

      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
        source_file_dependencies: diffusion_qwen_image_accuracy
        timeout_in_minutes: 180
        commands:
          - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model

      - label: ":full_moon: JoyAI-Image-Edit · Accuracy Test"
        source_file_dependencies: diffusion_joy_image_accuracy
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/test_joy_image_edit.py -m "full_model and diffusion and H100 and B200 and cards_1" --run-level full_model

      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
        source_file_dependencies: diffusion_hunyuan_image3_accuracy
        timeout_in_minutes: 180
        commands:
          - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
          - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py tests/e2e/accuracy/test_hunyuan_image3_lora.py -m "full_model and H100 and B200 and cards_4" --run-level full_model

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
        source_file_dependencies: diffusion_qwen_image_perf
        key: nightly-diffusion-x2iat-performance-qwen-image-single
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/diffusion_result_*.json
          - tests/dfx/perf/results/logs/*.log
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export CACHE_DIT_VERSION=1.5.0
          - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1"

      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
        source_file_dependencies: diffusion_qwen_image_perf
        key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/diffusion_result_*.json
          - tests/dfx/perf/results/logs/*.log
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export CACHE_DIT_VERSION=1.5.0
          # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN;
          # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM.
          - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4"

  - group: ":card_index_dividers: Diffusion X2V Model Test"
    key: nightly-diffusion-x2v-group
    depends_on: upload-nightly-pipeline
    if: >-
      build.env("NIGHTLY") == "1" ||
      build.pull_request.labels includes "nightly-test"
    steps:

      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
        source_file_dependencies: diffusion_hunyuan_video_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · LingBot Function Test"
        source_file_dependencies: diffusion_lingbot_function
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -s -v
            tests/e2e/online_serving/test_lingbot_video_expansion.py
            tests/e2e/online_serving/test_lingbot_video_moe.py
            tests/e2e/offline_inference/test_lingbot_world_v2.py
            -m "full_model and diffusion and H100 and B200 and cards_1"
            --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Doc Test"
        source_file_dependencies: diffusion_image_to_video_doc
        timeout_in_minutes: 60
        commands:
          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test"
        source_file_dependencies: diffusion_wan22_function
        timeout_in_minutes: 90
        commands:
          - |
            VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
              pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model"

      - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
        source_file_dependencies: diffusion_wan22_accuracy
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model

      - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
        source_file_dependencies: diffusion_minimax_h3_accuracy
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model

      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test"
        source_file_dependencies: diffusion_hunyuan_video_accuracy
        timeout_in_minutes: 180
        commands:
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model

      - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
        source_file_dependencies: diffusion_wan22_perf
        key: nightly-diffusion-x2v-performance-single
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1"

      - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
        source_file_dependencies:
          - diffusion_wan22_perf
          - diffusion_cosmos3_perf
        key: nightly-diffusion-x2v-performance-2gpu
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2"
            EXIT1=$$?
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2"
            EXIT2=$$?
            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
            exit $$EXIT2

      - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
        source_file_dependencies: diffusion_hunyuan_video_perf
        key: nightly-diffusion-x2v-performance-hunyuanvideo15
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
            EXIT1=$$?
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)"
            EXIT2=$$?
            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
            exit $$EXIT2

      - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
        source_file_dependencies: diffusion_minimax_h3_perf
        key: nightly-diffusion-x2v-performance-minimax-h3
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4"

      - label: ":full_moon: Diffusion X2V · Perf Test · LingBot"
        source_file_dependencies: diffusion_lingbot_perf
        key: nightly-diffusion-x2v-performance-lingbot
        timeout_in_minutes: 180
        artifact_paths:
          - tests/dfx/perf/results/*.json
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json -m "H100 and B200 and cards_1"

  - group: ":card_index_dividers: Diffusion Test"
    key: nightly-diffusion-test-group
    depends_on: upload-nightly-pipeline
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
    steps:
      - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
        source_file_dependencies: diffusion_distributed_attention
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model"

      - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
        source_file_dependencies: diffusion_distributed_attention
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model"

      - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
        source_file_dependencies: diffusion_distributed_attention
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model"

      - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
        source_file_dependencies: diffusion_distributed_attention
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"

      - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
        source_file_dependencies: diffusion_distributed_attention
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model"

      - label: ":full_moon: Diffusion · Offloader Test"
        source_file_dependencies: diffusion_offloader
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1"

      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
        source_file_dependencies: diffusion_quantization
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model"

      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
        source_file_dependencies: diffusion_minimax_h3_function
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model"

      - label: ":full_moon: Diffusion · Quantization Test with L4"
        source_file_dependencies: diffusion_quantization
        timeout_in_minutes: 60
        commands:
          - pip install "vllm-gguf-plugin==0.0.4"
          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model"

  # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
  # Do not add new depends_on entries or extend this step.
  - label: ":email: Nightly Collection & Email"
    key: nightly-perf-distribution
    depends_on:
      - nightly-omni-performance-no-async-chunk
      - nightly-omni-performance-async-chunk
      - nightly-tts-performance-single
      - nightly-tts-performance-2gpu
      - nightly-diffusion-x2iat-performance-qwen-image-single
      - nightly-diffusion-x2iat-performance-qwen-image-4gpu
      - nightly-diffusion-x2v-performance-single
      - nightly-diffusion-x2v-performance-2gpu
    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
    artifact_paths:
      - tests/dfx/perf/results/*.xlsx
      - tests/dfx/perf/results/*.html
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
      - python tools/nightly/generate_nightly_perf_excel.py
      - python tools/nightly/generate_nightly_perf_html.py
      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
    agents:
      queue: "cpu_queue_premerge"