vllm-omni/.buildkite/amd/test-amd-nightly.yml

# AMD MI300 `full_model` nightly coverage during burn-in.
# All jobs are non-blocking until the lane has stable runtime evidence.
# Historical build #11646 was red in Function Expansion and Accuracy (one
# async_chunk similarity mismatch; Seed-TTS WER 0.376480 > 0.35).

env:
  VLLM_WORKER_MULTIPROC_METHOD: spawn
  HF_HUB_DOWNLOAD_TIMEOUT: 300
  HF_HUB_ETAG_TIMEOUT: 60

steps:
  # MI325 is the repository's legacy ROCm pytest marker. The AMD template maps
  # agent_pool to the current MI300 Buildkite queues independently of that mark.
  - group: ":full_moon: Qwen3-Omni MI300 Nightly"
    steps:
      - label: "Qwen3-Omni Function Expansion"
        agent_pool: mi300_2
        depends_on: amd-build
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 180
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

      - label: "Qwen3-Omni Accuracy"
        agent_pool: mi300_2
        depends_on: amd-build
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 180
        artifact_paths:
          - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json
        commands:
          - export SEED_TTS_WER_EVAL=1
          # PyTorch ROCm uses the torch.cuda device namespace.
          - export SEED_TTS_EVAL_DEVICE=cuda:1
          - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"
          # Native tests run from the image copy under /app/vllm-omni, while
          # Buildkite's artifact hook searches the checkout workspace.
          - |
            if [[ "$$PWD" != "$${BUILDKITE_BUILD_CHECKOUT_PATH:?}" ]]; then
              artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH}/tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc"
              mkdir -p "$$artifact_dir"
              cp -v tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json "$$artifact_dir/"
            fi

      - label: "Qwen3-Omni Documentation Examples"
        agent_pool: mi300_2
        depends_on: amd-build
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 120
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

      - label: "Qwen3-Omni AITER-on Smoke"
        agent_pool: mi300_2
        depends_on: amd-build
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 90
        commands:
          - export VLLM_ROCM_USE_AITER=1
          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py::test_text_to_text_audio_001 -m "full_model and rocm and MI325 and cards_2" --run-level "full_model"

  - group: ":speaker: CosyVoice3 MI300 Nightly"
    steps:
      # Exercise all sync/async and Chinese/English cases with the platform's
      # default attention backend. On ROCm this retains end-to-end AITER
      # coverage without putting its cold JIT build on the READY critical path.
      - label: "CosyVoice3-TTS E2E Test"
        agent_pool: mi300_1
        depends_on: amd-build
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 90
        retry:
          automatic:
            - exit_status: 134
              limit: 1
        commands:
          - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"

  - group: ":full_moon: LingBot-Video MI300 Nightly"
    steps:
      - label: "LingBot-Video Dense CFG-off Smoke"
        agent_pool: mi300_1
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 90
        artifact_paths:
          - artifacts/rocm-lingbot-nightly/**/*
        commands:
          - pytest --collect-only -q tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model
          - pytest -s -v tests/e2e/online_serving/test_lingbot_video_expansion.py::test_cfg_off -m "full_model and diffusion and rocm and MI325 and cards_1" --run-level full_model --junitxml=/tmp/lingbot-pytest.xml
          - |
            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-lingbot-nightly"
            mkdir -p "$$artifact_dir"
            cp -v /tmp/lingbot-pytest.xml "$$artifact_dir/"

  - group: ":full_moon: Cosmos3 MI300 Nightly"
    steps:
      - label: "Cosmos3 T2I Function"
        agent_pool: mi300_1
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 90
        artifact_paths:
          - artifacts/rocm-cosmos3-nightly/**/*
        commands:
          # Cosmos3 uses both masked and unmasked BF16 AITER MHA modules. Build
          # them before server startup, then reuse the same job-local cache
          # during the smoke test.
          - export VLLM_ROCM_USE_AITER=1
          - export AITER_JIT_DIR="/tmp/vllm-omni-cosmos3-aiter-$$BUILDKITE_JOB_ID"
          - export TORCH_EXTENSIONS_DIR="/tmp/vllm-omni-cosmos3-torch-extensions-$$BUILDKITE_JOB_ID"
          - mkdir -p "$$AITER_JIT_DIR" "$$TORCH_EXTENSIONS_DIR"
          - pytest --collect-only -q tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001 -m "core_model and rocm and MI325 and cards_1" --run-level full_model
          - >-
            timeout --signal=TERM --kill-after=2m 55m
            python .buildkite/amd/scripts/prewarm_cosmos3_aiter.py
          - >-
            timeout --signal=TERM --kill-after=2m 25m
            pytest -s -v
            tests/e2e/online_serving/test_cosmos3.py::test_text_to_image_001
            -m "core_model and rocm and MI325 and cards_1"
            --run-level full_model
            --junitxml=/tmp/cosmos3-pytest.xml
          - |
            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-cosmos3-nightly"
            mkdir -p "$$artifact_dir"
            cp -v /tmp/cosmos3-pytest.xml "$$artifact_dir/"

  - group: ":full_moon: HunyuanImage3 MI300 Nightly"
    steps:
      - label: "HunyuanImage3 Offline Pixel Accuracy"
        agent_pool: mi300_4
        mirror_hardwares: [amdproduction]
        grade: NonBlocking
        timeout_in_minutes: 180
        env:
          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
        artifact_paths:
          - artifacts/rocm-hunyuanimage3/**/*
        commands:
          - pytest --collect-only -q tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model
          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model --junitxml=/tmp/hunyuanimage3-pytest.xml
          - |
            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-hunyuanimage3"
            mkdir -p "$$artifact_dir"
            cp -v tests/e2e/accuracy/artifacts/HunyuanImage-3_0-Instruct/*.png "$$artifact_dir/"
            cp -v /tmp/hunyuanimage3-pytest.xml "$$artifact_dir/"