From e0fde88a5f582113963b1f3ee63997b780b13e9b Mon Sep 17 00:00:00 2001 From: TJian Date: Tue, 9 Jun 2026 22:17:55 +0800 Subject: [PATCH] [ROCm] [CI] Add group feature and envs feature (#4208) Signed-off-by: tjtanaa --- .buildkite/test-amd-merge.yml | 307 ++++++++++++++------------- .buildkite/test-amd-ready.yaml | 300 +++++++++++++------------- .buildkite/test-template-amd-omni.j2 | 82 ++++++- 3 files changed, 394 insertions(+), 295 deletions(-) diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index b986d687c..1a5217b84 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -1,54 +1,111 @@ +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + steps: -- label: "Simple · Diffusion & Model Executor Test" - agent_pool: mi325_1 - depends_on: amd-build - timeout_in_minutes: 50 - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_ROCM_USE_AITER=0 - # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" +- group: ":card_index_dividers: Simple Test" + steps: + - label: "Simple · Diffusion & Model Executor Test" + agent_pool: mi325_1 + depends_on: amd-build + timeout_in_minutes: 50 + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" -- label: "Simple · Other Test" - agent_pool: mi325_1 - depends_on: amd-build - timeout_in_minutes: 50 - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_ROCM_USE_AITER=0 - # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - label: "Simple · Other Test" + agent_pool: mi325_1 + depends_on: amd-build + timeout_in_minutes: 50 + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" -- label: "Diffusion Images API LoRA E2E" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py +- group: ":card_index_dividers: Diffusion Test" + steps: + - label: "Diffusion · LoRA Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py -- label: "Diffusion Model CPU offloading Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - | - timeout 20m bash -c ' - set +e - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py - EXIT1=\$? - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py - EXIT2=\$? - exit \$((EXIT1 | EXIT2)) - ' + - label: "Diffusion Model CPU offloading Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - | + timeout 20m bash -c ' + set +e + pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py + EXIT1=\$? + pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py + EXIT2=\$? + exit \$((EXIT1 | EXIT2)) + ' + + - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)" + agent_pool: mi325_4 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py + - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py + + # merge-only tests + - label: "Diffusion Tensor Parallelism Test" + agent_pool: mi325_2 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py + + - label: "Diffusion Model Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py + + - label: "Diffusion GPU Worker Test" + agent_pool: mi325_2 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py + + - label: "Diffusion · Wan22 Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274 # - label: "Audio Generation Model Test" @@ -62,42 +119,6 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model -- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)" - agent_pool: mi325_4 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py - - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py - -# merge-only tests -- label: "Diffusion Tensor Parallelism Test" - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py - -- label: "Diffusion Model Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py - -- label: "Diffusion GPU Worker Test" - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py - - label: "Engine Test" agent_pool: mi325_1 depends_on: amd-build @@ -111,50 +132,65 @@ steps: # This model seems to be important, so I am trying to run it here. # Since this is from test-nightly.yml, # I keep this non-blocking. -- label: "Omni Model Test Qwen2-5-Omni" - agent_pool: mi325_4 - depends_on: amd-build - mirror_hardwares: [amdproduction] - commands: - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" +- group: ":card_index_dividers: E2E Test" + steps: + - label: "Omni Model Test Qwen2-5-Omni" + agent_pool: mi325_4 + depends_on: amd-build + mirror_hardwares: [amdproduction] + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" -- label: "Omni Model Test Qwen3-Omni" - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' + - label: "Omni Model Test Qwen3-Omni" + agent_pool: mi325_2 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model' -- label: "Qwen3-TTS CustomVoice E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 20m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" - ' + - label: "Qwen3-TTS CustomVoice E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 20m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" + ' -- label: "Qwen3-TTS Base E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 30m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" - ' + - label: "Qwen3-TTS Base E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 30m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" + ' + + - label: "Voxtral-TTS E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 20m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" + ' # TODO: Bagel test on ROCm is very unstable. @tjtanaa # Need to debug before reneable numerical changes across large PRs @@ -195,28 +231,3 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 # - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm" - -- label: "Diffusion · Wan22 Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 40m bash -c " - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' - " - -- label: "Voxtral-TTS E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 20m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - ' diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/test-amd-ready.yaml index 9951544bd..9da9507c8 100644 --- a/.buildkite/test-amd-ready.yaml +++ b/.buildkite/test-amd-ready.yaml @@ -1,42 +1,33 @@ +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + steps: -- label: "Simple · Diffusion & Model Executor Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_ROCM_USE_AITER=0 - # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" +- group: ":card_index_dividers: Simple Test" + steps: + - label: "Simple · Diffusion & Model Executor Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" -- label: "Simple · Other Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_ROCM_USE_AITER=0 - # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - # that is not supported on CPU - - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - -- label: "Voxtral TTS CUDA Unit Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py - -- label: "Diffusion · Batch Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" + - label: "Simple · Other Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_ROCM_USE_AITER=0 + # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM + # that is not supported on CPU + - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - label: "Custom Pipeline Test" agent_pool: mi325_1 @@ -46,23 +37,92 @@ steps: commands: - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model" -- label: "Diffusion Model CPU offloading Test" + +- label: "Voxtral TTS CUDA Unit Test" agent_pool: mi325_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - | - timeout 20m bash -c ' - set +e - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py - EXIT1=\$? - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py - EXIT2=\$? - exit \$((EXIT1 | EXIT2)) - ' + - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py + +- group: ":card_index_dividers: Diffusion Test" + steps: + - label: "Diffusion · Batch Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" + + - label: "Diffusion · Offloader Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - | + timeout 20m bash -c ' + set +e + pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py + EXIT1=\$? + pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py + EXIT2=\$? + exit \$((EXIT1 | EXIT2)) + ' + + - label: "Diffusion · Cache Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export GPU_ARCHS=gfx942 + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4" + + - label: "Diffusion Sequence Parallelism Test" + agent_pool: mi325_2 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model + + - label: "Diffusion GPU Worker Test" + agent_pool: mi325_2 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py + + - label: "Diffusion · Model Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" + + - label: "Diffusion · Wan22 Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 40m bash -c " + export VLLM_TEST_CLEAN_GPU_MEMORY=1 + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' + " ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274 # - label: "Audio Generation Model Test" @@ -76,56 +136,6 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model -- label: "Diffusion Cache Backend Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export GPU_ARCHS=gfx942 - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4" - -- label: "Diffusion Sequence Parallelism Test" - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model - -- label: "Diffusion GPU Worker Test" - agent_pool: mi325_2 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py - -- label: "Diffusion · Model Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" - -- label: "Diffusion · Wan22 Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 40m bash -c " - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' - " - - label: "Engine Test" agent_pool: mi325_1 depends_on: amd-build @@ -149,43 +159,56 @@ steps: # - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py # - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model" -- label: "MiMo-Audio E2E Test with H100" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 30m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model" - ' +- group: ":card_index_dividers: E2E Test" + steps: + - label: "MiMo-Audio E2E Test with H100" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 30m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model" + ' -- label: "Qwen3-TTS E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model" + - label: "Qwen3-TTS E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model" -- label: "Voxtral-TTS E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 20m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - ' + - label: "Voxtral-TTS E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 20m bash -c ' + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" + pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" + ' + + - label: "CosyVoice3-TTS E2E Test" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 20m bash -c ' + pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model" + ' # TODO: Bagel test on ROCm is very unstable. @tjtanaa # Need to debug before reneable numerical changes across large PRs @@ -225,14 +248,3 @@ steps: # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 # - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm" - -- label: "CosyVoice3-TTS E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 20m bash -c ' - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model" - ' diff --git a/.buildkite/test-template-amd-omni.j2 b/.buildkite/test-template-amd-omni.j2 index db76f8216..aa5ca8bcd 100644 --- a/.buildkite/test-template-amd-omni.j2 +++ b/.buildkite/test-template-amd-omni.j2 @@ -6,6 +6,13 @@ {% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %} {% set default_working_dir = "/app/vllm-omni" %} +{% if env %} +env: +{% for name in env %} + {{ name }}: "{{ env[name] }}" +{% endfor %} +{% endif %} +steps: - group: "AMD Tests" depends_on: ~ steps: @@ -31,8 +38,15 @@ agents: queue: amd-cpu - {% for step in steps %} - {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %} +{% for step_group in steps %} +{% if step_group.group %} + - group: "{{ step_group.group }}" +{% if step_group.depends_on %} + depends_on: {{ step_group.depends_on }} +{% endif %} + steps: +{% for step in step_group.steps %} +{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %} - label: "{{ step.agent_pool }}: {{ step.label }}" depends_on: amd-build agents: @@ -46,8 +60,23 @@ command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh env: DOCKER_BUILDKIT: "1" +{% if step.env %} +{% for name in step.env %} + {{ name }}: "{{ step.env[name] }}" +{% endfor %} +{% endif %} TEST_COMMAND: |- (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }} +{% if env %} +{% for name in env %} + export {{ name }}="{{ env[name] }}" +{% endfor %} +{% endif %} +{% if step.env %} +{% for name in step.env %} + export {{ name }}="{{ step.env[name] }}" +{% endfor %} +{% endif %} {% if "mi250" in step.agent_pool %} python3 -m pip uninstall -y amd-aiter {% endif %} @@ -61,5 +90,52 @@ {% else %} soft_fail: true {% endif%} +{% endif %} +{% endfor %} +{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %} +{% set step = step_group %} + - label: "{{ step.agent_pool }}: {{ step.label }}" + depends_on: amd-build + agents: + {% if step.agent_pool %} + queue: amd_{{ step.agent_pool }} + {% else %} + queue: amd_mi325_1 + {% endif %} +{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %} +{% set indented_cmd = cmd_body | replace("\n", "\n ") %} + command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh + env: + DOCKER_BUILDKIT: "1" +{% if step.env %} +{% for name in step.env %} + {{ name }}: "{{ step.env[name] }}" +{% endfor %} +{% endif %} + TEST_COMMAND: |- + (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }} +{% if env %} +{% for name in env %} + export {{ name }}="{{ env[name] }}" +{% endfor %} +{% endif %} +{% if step.env %} +{% for name in step.env %} + export {{ name }}="{{ step.env[name] }}" +{% endfor %} +{% endif %} +{% if "mi250" in step.agent_pool %} + python3 -m pip uninstall -y amd-aiter +{% endif %} + {{ indented_cmd | safe }} + priority: 100 + {% if step.timeout_in_minutes %} + timeout_in_minutes: {{ step.timeout_in_minutes }} {% endif %} - {% endfor %} + {% if step.grade and step.grade == "Blocking" %} + soft_fail: false + {% else %} + soft_fail: true + {% endif%} +{% endif %} +{% endfor %}