[ROCm] [CI] [Bugfix] 2/N Fix Qwen2.5 and Qwen3 test (#3343)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>

.buildkite/test-amd-merge.yml

@@ -105,16 +105,17 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
+# This is from test-nightly.yml.
+# This model seems to be important, so I am trying to run it here.
+# Since this is from test-nightly.yml,
+# I keep this non-blocking.
 - label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_2
+  agent_pool: mi325_4
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
-  grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2
@@ -122,10 +123,9 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Qwen3-TTS CustomVoice E2E Test"
   agent_pool: mi325_1

tests/e2e/offline_inference/test_qwen3_omni.py

@@ -34,7 +34,7 @@ def get_cuda_graph_config():
     )
 
 
-if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
+if current_omni_platform.is_xpu():
     stage_configs = [_CI_DEPLOY]
 else:
     stage_configs = [get_cuda_graph_config()]

tests/helpers/env.py

@@ -135,54 +135,97 @@ def wait_for_gpu_memory_to_clear(
 
 
 def _print_gpu_processes() -> None:
-    """Print GPU information including nvidia-smi and system processes."""
+    """Print GPU information including nvidia-smi/amd-smi and system processes."""
+    from vllm.platforms import current_platform
 
-    print("\n" + "=" * 80)
+    # Check for NVIDIA GPUs
-    print("NVIDIA GPU Information (nvidia-smi)")
+    if current_platform.is_cuda():
-    print("=" * 80)
+        print("\n" + "=" * 80)
+        print("NVIDIA GPU Information (nvidia-smi)")
+        print("=" * 80)
+        try:
+            nvidia_result = subprocess.run(
+                ["nvidia-smi"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if nvidia_result.returncode == 0:
+                lines = nvidia_result.stdout.strip().split("\n")
+                for line in lines[:20]:
+                    print(line)
+                if len(lines) > 20:
+                    print(f"... (showing first 20 of {len(lines)} lines)")
+            else:
+                print("nvidia-smi command failed")
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            print("nvidia-smi not available or timed out")
+        except Exception as e:
+            print(f"Error running nvidia-smi: {e}")
 
-    try:
+        print("\n" + "=" * 80)
-        nvidia_result = subprocess.run(
+        print("Detailed GPU Processes (nvidia-smi pmon)")
-            ["nvidia-smi"],
+        print("=" * 80)
-            capture_output=True,
+        try:
-            text=True,
+            pmon_result = subprocess.run(
-            timeout=5,
+                ["nvidia-smi", "pmon", "-c", "1"],
-        )
+                capture_output=True,
+                text=True,
+                timeout=3,
+            )
+            if pmon_result.returncode == 0 and pmon_result.stdout.strip():
+                print(pmon_result.stdout)
+            else:
+                print("No active GPU processes found via nvidia-smi pmon")
+        except Exception:
+            print("nvidia-smi pmon not available")
 
-        if nvidia_result.returncode == 0:
+    # Check for AMD ROCm GPUs
-            lines = nvidia_result.stdout.strip().split("\n")
+    elif current_platform.is_rocm():
-            for line in lines[:20]:
+        print("\n" + "=" * 80)
-                print(line)
+        print("AMD GPU Information (amd-smi)")
+        print("=" * 80)
+        try:
+            amd_result = subprocess.run(
+                ["amd-smi"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if amd_result.returncode == 0:
+                lines = amd_result.stdout.strip().split("\n")
+                for line in lines[:30]:  # Show more lines for AMD output
+                    print(line)
+                if len(lines) > 30:
+                    print(f"... (showing first 30 of {len(lines)} lines)")
+            else:
+                print("amd-smi command failed")
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            print("amd-smi not available or timed out")
+        except Exception as e:
+            print(f"Error running amd-smi: {e}")
 
-            if len(lines) > 20:
+        print("\n" + "=" * 80)
-                print(f"... (showing first 20 of {len(lines)} lines)")
+        print("Detailed AMD GPU Processes (amd-smi process)")
-        else:
+        print("=" * 80)
-            print("nvidia-smi command failed")
+        try:
+            process_result = subprocess.run(
+                ["amd-smi", "process"],
+                capture_output=True,
+                text=True,
+                timeout=3,
+            )
+            if process_result.returncode == 0 and process_result.stdout.strip():
+                print(process_result.stdout)
+            else:
+                print("No active GPU processes found via amd-smi process")
+        except Exception:
+            print("amd-smi process not available")
 
-    except (subprocess.TimeoutExpired, FileNotFoundError):
+    else:
-        print("nvidia-smi not available or timed out")
+        print("\n" + "=" * 80)
-    except Exception as e:
+        print("WARNING: No supported GPU platform detected (neither CUDA nor ROCm)")
-        print(f"Error running nvidia-smi: {e}")
+        print("=" * 80)
-
-    print("\n" + "=" * 80)
-    print("Detailed GPU Processes (nvidia-smi pmon)")
-    print("=" * 80)
-
-    try:
-        pmon_result = subprocess.run(
-            ["nvidia-smi", "pmon", "-c", "1"],
-            capture_output=True,
-            text=True,
-            timeout=3,
-        )
-
-        if pmon_result.returncode == 0 and pmon_result.stdout.strip():
-            print(pmon_result.stdout)
-        else:
-            print("No active GPU processes found via nvidia-smi pmon")
-
-    except Exception:
-        print("nvidia-smi pmon not available")
 
     print("\n" + "=" * 80)
     print("System Processes with GPU keywords")

tests/helpers/stage_config.py

@@ -298,13 +298,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
             },
         ],
         "platforms": {
-            "rocm": {
-                "stages": [
-                    {"stage_id": 0, "gpu_memory_utilization": 0.9},
-                    {"stage_id": 1, "gpu_memory_utilization": 0.4},
-                    {"stage_id": 2, "gpu_memory_utilization": 0.5, "devices": "2"},
-                ],
-            },
             "xpu": {
                 "stages": [
                     {
@@ -354,23 +347,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
             },
         ],
         "platforms": {
-            "rocm": {
-                "stages": [
-                    {"stage_id": 0, "max_num_seqs": 1, "default_sampling_params": {"max_tokens": 100}},
-                    {
-                        "stage_id": 1,
-                        "max_num_seqs": 1,
-                        "enforce_eager": True,
-                        "default_sampling_params": {"max_tokens": 100},
-                    },
-                    {
-                        "stage_id": 2,
-                        "max_num_seqs": 1,
-                        "max_num_batched_tokens": 1000000,
-                        "default_sampling_params": {"max_tokens": 200},
-                    },
-                ],
-            },
             "xpu": {
                 "stages": [
                     {

vllm_omni/deploy/qwen2_5_omni.yaml

@@ -71,12 +71,6 @@ platforms:
         # 3-NPU layout: stage 2 lives on its own card.
         devices: "2"
 
-  rocm:
-    stages:
-      - stage_id: 2
-        # 3-GPU MI325 layout: stage 2 on a separate card.
-        devices: "2"
-
   xpu:
     stages:
       # Verified on 2x Intel Arc Pro B60. Both AR stages use cudagraphs.

vllm_omni/deploy/qwen3_omni_moe.yaml

@@ -76,11 +76,6 @@ platforms:
         gpu_memory_utilization: 0.3
         devices: "2"
 
-  rocm:
-    stages:
-      - stage_id: 0
-        enforce_eager: true
-
   xpu:
     stages:
       - stage_id: 0