diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index aabff610d..93013a52a 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -105,16 +105,17 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py +# This is from test-nightly.yml. +# This model seems to be important, so I am trying to run it here. +# Since this is from test-nightly.yml, +# I keep this non-blocking. - label: "Omni Model Test Qwen2-5-Omni" - agent_pool: mi325_2 + agent_pool: mi325_4 depends_on: amd-build mirror_hardwares: [amdproduction] - grade: Blocking commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py - - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model" + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" - label: "Omni Model Test Qwen3-Omni" agent_pool: mi325_2 @@ -122,10 +123,9 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking commands: - - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model" + - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model" - label: "Qwen3-TTS CustomVoice E2E Test" agent_pool: mi325_1 diff --git a/tests/e2e/offline_inference/test_qwen3_omni.py b/tests/e2e/offline_inference/test_qwen3_omni.py index c4d257b51..f599f2a78 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni.py +++ b/tests/e2e/offline_inference/test_qwen3_omni.py @@ -34,7 +34,7 @@ def get_cuda_graph_config(): ) -if current_omni_platform.is_rocm() or current_omni_platform.is_xpu(): +if current_omni_platform.is_xpu(): stage_configs = [_CI_DEPLOY] else: stage_configs = [get_cuda_graph_config()] diff --git a/tests/helpers/env.py b/tests/helpers/env.py index 22ec9a786..8df78e09e 100644 --- a/tests/helpers/env.py +++ b/tests/helpers/env.py @@ -135,54 +135,97 @@ def wait_for_gpu_memory_to_clear( def _print_gpu_processes() -> None: - """Print GPU information including nvidia-smi and system processes.""" + """Print GPU information including nvidia-smi/amd-smi and system processes.""" + from vllm.platforms import current_platform - print("\n" + "=" * 80) - print("NVIDIA GPU Information (nvidia-smi)") - print("=" * 80) + # Check for NVIDIA GPUs + if current_platform.is_cuda(): + print("\n" + "=" * 80) + print("NVIDIA GPU Information (nvidia-smi)") + print("=" * 80) + try: + nvidia_result = subprocess.run( + ["nvidia-smi"], + capture_output=True, + text=True, + timeout=5, + ) + if nvidia_result.returncode == 0: + lines = nvidia_result.stdout.strip().split("\n") + for line in lines[:20]: + print(line) + if len(lines) > 20: + print(f"... (showing first 20 of {len(lines)} lines)") + else: + print("nvidia-smi command failed") + except (subprocess.TimeoutExpired, FileNotFoundError): + print("nvidia-smi not available or timed out") + except Exception as e: + print(f"Error running nvidia-smi: {e}") - try: - nvidia_result = subprocess.run( - ["nvidia-smi"], - capture_output=True, - text=True, - timeout=5, - ) + print("\n" + "=" * 80) + print("Detailed GPU Processes (nvidia-smi pmon)") + print("=" * 80) + try: + pmon_result = subprocess.run( + ["nvidia-smi", "pmon", "-c", "1"], + capture_output=True, + text=True, + timeout=3, + ) + if pmon_result.returncode == 0 and pmon_result.stdout.strip(): + print(pmon_result.stdout) + else: + print("No active GPU processes found via nvidia-smi pmon") + except Exception: + print("nvidia-smi pmon not available") - if nvidia_result.returncode == 0: - lines = nvidia_result.stdout.strip().split("\n") - for line in lines[:20]: - print(line) + # Check for AMD ROCm GPUs + elif current_platform.is_rocm(): + print("\n" + "=" * 80) + print("AMD GPU Information (amd-smi)") + print("=" * 80) + try: + amd_result = subprocess.run( + ["amd-smi"], + capture_output=True, + text=True, + timeout=5, + ) + if amd_result.returncode == 0: + lines = amd_result.stdout.strip().split("\n") + for line in lines[:30]: # Show more lines for AMD output + print(line) + if len(lines) > 30: + print(f"... (showing first 30 of {len(lines)} lines)") + else: + print("amd-smi command failed") + except (subprocess.TimeoutExpired, FileNotFoundError): + print("amd-smi not available or timed out") + except Exception as e: + print(f"Error running amd-smi: {e}") - if len(lines) > 20: - print(f"... (showing first 20 of {len(lines)} lines)") - else: - print("nvidia-smi command failed") + print("\n" + "=" * 80) + print("Detailed AMD GPU Processes (amd-smi process)") + print("=" * 80) + try: + process_result = subprocess.run( + ["amd-smi", "process"], + capture_output=True, + text=True, + timeout=3, + ) + if process_result.returncode == 0 and process_result.stdout.strip(): + print(process_result.stdout) + else: + print("No active GPU processes found via amd-smi process") + except Exception: + print("amd-smi process not available") - except (subprocess.TimeoutExpired, FileNotFoundError): - print("nvidia-smi not available or timed out") - except Exception as e: - print(f"Error running nvidia-smi: {e}") - - print("\n" + "=" * 80) - print("Detailed GPU Processes (nvidia-smi pmon)") - print("=" * 80) - - try: - pmon_result = subprocess.run( - ["nvidia-smi", "pmon", "-c", "1"], - capture_output=True, - text=True, - timeout=3, - ) - - if pmon_result.returncode == 0 and pmon_result.stdout.strip(): - print(pmon_result.stdout) - else: - print("No active GPU processes found via nvidia-smi pmon") - - except Exception: - print("nvidia-smi pmon not available") + else: + print("\n" + "=" * 80) + print("WARNING: No supported GPU platform detected (neither CUDA nor ROCm)") + print("=" * 80) print("\n" + "=" * 80) print("System Processes with GPU keywords") diff --git a/tests/helpers/stage_config.py b/tests/helpers/stage_config.py index 66a58378d..310af9174 100644 --- a/tests/helpers/stage_config.py +++ b/tests/helpers/stage_config.py @@ -298,13 +298,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = { }, ], "platforms": { - "rocm": { - "stages": [ - {"stage_id": 0, "gpu_memory_utilization": 0.9}, - {"stage_id": 1, "gpu_memory_utilization": 0.4}, - {"stage_id": 2, "gpu_memory_utilization": 0.5, "devices": "2"}, - ], - }, "xpu": { "stages": [ { @@ -354,23 +347,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = { }, ], "platforms": { - "rocm": { - "stages": [ - {"stage_id": 0, "max_num_seqs": 1, "default_sampling_params": {"max_tokens": 100}}, - { - "stage_id": 1, - "max_num_seqs": 1, - "enforce_eager": True, - "default_sampling_params": {"max_tokens": 100}, - }, - { - "stage_id": 2, - "max_num_seqs": 1, - "max_num_batched_tokens": 1000000, - "default_sampling_params": {"max_tokens": 200}, - }, - ], - }, "xpu": { "stages": [ { diff --git a/vllm_omni/deploy/qwen2_5_omni.yaml b/vllm_omni/deploy/qwen2_5_omni.yaml index 41aef0df6..7ab87e590 100644 --- a/vllm_omni/deploy/qwen2_5_omni.yaml +++ b/vllm_omni/deploy/qwen2_5_omni.yaml @@ -71,12 +71,6 @@ platforms: # 3-NPU layout: stage 2 lives on its own card. devices: "2" - rocm: - stages: - - stage_id: 2 - # 3-GPU MI325 layout: stage 2 on a separate card. - devices: "2" - xpu: stages: # Verified on 2x Intel Arc Pro B60. Both AR stages use cudagraphs. diff --git a/vllm_omni/deploy/qwen3_omni_moe.yaml b/vllm_omni/deploy/qwen3_omni_moe.yaml index 39baed6bd..bbc8e1140 100644 --- a/vllm_omni/deploy/qwen3_omni_moe.yaml +++ b/vllm_omni/deploy/qwen3_omni_moe.yaml @@ -76,11 +76,6 @@ platforms: gpu_memory_utilization: 0.3 devices: "2" - rocm: - stages: - - stage_id: 0 - enforce_eager: true - xpu: stages: - stage_id: 0