[ROCm] [CI] [Bugfix] 2/N Fix Qwen2.5 and Qwen3 test (#3343)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian authored and GitHub committed 2026-05-05 23:28:37 +08:00
1 parent fde38ca54f
commit a77c56725d
6 files changed
+95 -87

No files matched your search

@@ -34,7 +34,7 @@ def get_cuda_graph_config():
)
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
if current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else:
stage_configs = [get_cuda_graph_config()]
+86 -43
View File
@@ -135,54 +135,97 @@ def wait_for_gpu_memory_to_clear(
def _print_gpu_processes() -> None:
"""Print GPU information including nvidia-smi and system processes."""
"""Print GPU information including nvidia-smi/amd-smi and system processes."""
from vllm.platforms import current_platform
print("\n" + "=" * 80)
print("NVIDIA GPU Information (nvidia-smi)")
print("=" * 80)
# Check for NVIDIA GPUs
if current_platform.is_cuda():
print("\n" + "=" * 80)
print("NVIDIA GPU Information (nvidia-smi)")
print("=" * 80)
try:
nvidia_result = subprocess.run(
["nvidia-smi"],
capture_output=True,
text=True,
timeout=5,
)
if nvidia_result.returncode == 0:
lines = nvidia_result.stdout.strip().split("\n")
for line in lines[:20]:
print(line)
if len(lines) > 20:
print(f"... (showing first 20 of {len(lines)} lines)")
else:
print("nvidia-smi command failed")
except (subprocess.TimeoutExpired, FileNotFoundError):
print("nvidia-smi not available or timed out")
except Exception as e:
print(f"Error running nvidia-smi: {e}")
try:
nvidia_result = subprocess.run(
["nvidia-smi"],
capture_output=True,
text=True,
timeout=5,
)
print("\n" + "=" * 80)
print("Detailed GPU Processes (nvidia-smi pmon)")
print("=" * 80)
try:
pmon_result = subprocess.run(
["nvidia-smi", "pmon", "-c", "1"],
capture_output=True,
text=True,
timeout=3,
)
if pmon_result.returncode == 0 and pmon_result.stdout.strip():
print(pmon_result.stdout)
else:
print("No active GPU processes found via nvidia-smi pmon")
except Exception:
print("nvidia-smi pmon not available")
if nvidia_result.returncode == 0:
lines = nvidia_result.stdout.strip().split("\n")
for line in lines[:20]:
print(line)
# Check for AMD ROCm GPUs
elif current_platform.is_rocm():
print("\n" + "=" * 80)
print("AMD GPU Information (amd-smi)")
print("=" * 80)
try:
amd_result = subprocess.run(
["amd-smi"],
capture_output=True,
text=True,
timeout=5,
)
if amd_result.returncode == 0:
lines = amd_result.stdout.strip().split("\n")
for line in lines[:30]: # Show more lines for AMD output
print(line)
if len(lines) > 30:
print(f"... (showing first 30 of {len(lines)} lines)")
else:
print("amd-smi command failed")
except (subprocess.TimeoutExpired, FileNotFoundError):
print("amd-smi not available or timed out")
except Exception as e:
print(f"Error running amd-smi: {e}")
if len(lines) > 20:
print(f"... (showing first 20 of {len(lines)} lines)")
else:
print("nvidia-smi command failed")
print("\n" + "=" * 80)
print("Detailed AMD GPU Processes (amd-smi process)")
print("=" * 80)
try:
process_result = subprocess.run(
["amd-smi", "process"],
capture_output=True,
text=True,
timeout=3,
)
if process_result.returncode == 0 and process_result.stdout.strip():
print(process_result.stdout)
else:
print("No active GPU processes found via amd-smi process")
except Exception:
print("amd-smi process not available")
except (subprocess.TimeoutExpired, FileNotFoundError):
print("nvidia-smi not available or timed out")
except Exception as e:
print(f"Error running nvidia-smi: {e}")
print("\n" + "=" * 80)
print("Detailed GPU Processes (nvidia-smi pmon)")
print("=" * 80)
try:
pmon_result = subprocess.run(
["nvidia-smi", "pmon", "-c", "1"],
capture_output=True,
text=True,
timeout=3,
)
if pmon_result.returncode == 0 and pmon_result.stdout.strip():
print(pmon_result.stdout)
else:
print("No active GPU processes found via nvidia-smi pmon")
except Exception:
print("nvidia-smi pmon not available")
else:
print("\n" + "=" * 80)
print("WARNING: No supported GPU platform detected (neither CUDA nor ROCm)")
print("=" * 80)
print("\n" + "=" * 80)
print("System Processes with GPU keywords")
-24
View File
@@ -298,13 +298,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
},
],
"platforms": {
"rocm": {
"stages": [
{"stage_id": 0, "gpu_memory_utilization": 0.9},
{"stage_id": 1, "gpu_memory_utilization": 0.4},
{"stage_id": 2, "gpu_memory_utilization": 0.5, "devices": "2"},
],
},
"xpu": {
"stages": [
{
@@ -354,23 +347,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
},
],
"platforms": {
"rocm": {
"stages": [
{"stage_id": 0, "max_num_seqs": 1, "default_sampling_params": {"max_tokens": 100}},
{
"stage_id": 1,
"max_num_seqs": 1,
"enforce_eager": True,
"default_sampling_params": {"max_tokens": 100},
},
{
"stage_id": 2,
"max_num_seqs": 1,
"max_num_batched_tokens": 1000000,
"default_sampling_params": {"max_tokens": 200},
},
],
},
"xpu": {
"stages": [
{