[ROCm] [CI] [Bugfix] 2/N Fix Qwen2.5 and Qwen3 test (#3343)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
1 parent
fde38ca54f
commit
a77c56725d
6 files changed
+95
-87
No files matched your search
@@ -34,7 +34,7 @@ def get_cuda_graph_config():
|
||||
)
|
||||
|
||||
|
||||
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
|
||||
if current_omni_platform.is_xpu():
|
||||
stage_configs = [_CI_DEPLOY]
|
||||
else:
|
||||
stage_configs = [get_cuda_graph_config()]
|
||||
|
||||
+86
-43
@@ -135,54 +135,97 @@ def wait_for_gpu_memory_to_clear(
|
||||
|
||||
|
||||
def _print_gpu_processes() -> None:
|
||||
"""Print GPU information including nvidia-smi and system processes."""
|
||||
"""Print GPU information including nvidia-smi/amd-smi and system processes."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("NVIDIA GPU Information (nvidia-smi)")
|
||||
print("=" * 80)
|
||||
# Check for NVIDIA GPUs
|
||||
if current_platform.is_cuda():
|
||||
print("\n" + "=" * 80)
|
||||
print("NVIDIA GPU Information (nvidia-smi)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
nvidia_result = subprocess.run(
|
||||
["nvidia-smi"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if nvidia_result.returncode == 0:
|
||||
lines = nvidia_result.stdout.strip().split("\n")
|
||||
for line in lines[:20]:
|
||||
print(line)
|
||||
if len(lines) > 20:
|
||||
print(f"... (showing first 20 of {len(lines)} lines)")
|
||||
else:
|
||||
print("nvidia-smi command failed")
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
print("nvidia-smi not available or timed out")
|
||||
except Exception as e:
|
||||
print(f"Error running nvidia-smi: {e}")
|
||||
|
||||
try:
|
||||
nvidia_result = subprocess.run(
|
||||
["nvidia-smi"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
print("\n" + "=" * 80)
|
||||
print("Detailed GPU Processes (nvidia-smi pmon)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
pmon_result = subprocess.run(
|
||||
["nvidia-smi", "pmon", "-c", "1"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
)
|
||||
if pmon_result.returncode == 0 and pmon_result.stdout.strip():
|
||||
print(pmon_result.stdout)
|
||||
else:
|
||||
print("No active GPU processes found via nvidia-smi pmon")
|
||||
except Exception:
|
||||
print("nvidia-smi pmon not available")
|
||||
|
||||
if nvidia_result.returncode == 0:
|
||||
lines = nvidia_result.stdout.strip().split("\n")
|
||||
for line in lines[:20]:
|
||||
print(line)
|
||||
# Check for AMD ROCm GPUs
|
||||
elif current_platform.is_rocm():
|
||||
print("\n" + "=" * 80)
|
||||
print("AMD GPU Information (amd-smi)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
amd_result = subprocess.run(
|
||||
["amd-smi"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if amd_result.returncode == 0:
|
||||
lines = amd_result.stdout.strip().split("\n")
|
||||
for line in lines[:30]: # Show more lines for AMD output
|
||||
print(line)
|
||||
if len(lines) > 30:
|
||||
print(f"... (showing first 30 of {len(lines)} lines)")
|
||||
else:
|
||||
print("amd-smi command failed")
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
print("amd-smi not available or timed out")
|
||||
except Exception as e:
|
||||
print(f"Error running amd-smi: {e}")
|
||||
|
||||
if len(lines) > 20:
|
||||
print(f"... (showing first 20 of {len(lines)} lines)")
|
||||
else:
|
||||
print("nvidia-smi command failed")
|
||||
print("\n" + "=" * 80)
|
||||
print("Detailed AMD GPU Processes (amd-smi process)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
process_result = subprocess.run(
|
||||
["amd-smi", "process"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
)
|
||||
if process_result.returncode == 0 and process_result.stdout.strip():
|
||||
print(process_result.stdout)
|
||||
else:
|
||||
print("No active GPU processes found via amd-smi process")
|
||||
except Exception:
|
||||
print("amd-smi process not available")
|
||||
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
print("nvidia-smi not available or timed out")
|
||||
except Exception as e:
|
||||
print(f"Error running nvidia-smi: {e}")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("Detailed GPU Processes (nvidia-smi pmon)")
|
||||
print("=" * 80)
|
||||
|
||||
try:
|
||||
pmon_result = subprocess.run(
|
||||
["nvidia-smi", "pmon", "-c", "1"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
if pmon_result.returncode == 0 and pmon_result.stdout.strip():
|
||||
print(pmon_result.stdout)
|
||||
else:
|
||||
print("No active GPU processes found via nvidia-smi pmon")
|
||||
|
||||
except Exception:
|
||||
print("nvidia-smi pmon not available")
|
||||
else:
|
||||
print("\n" + "=" * 80)
|
||||
print("WARNING: No supported GPU platform detected (neither CUDA nor ROCm)")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("System Processes with GPU keywords")
|
||||
|
||||
@@ -298,13 +298,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
|
||||
},
|
||||
],
|
||||
"platforms": {
|
||||
"rocm": {
|
||||
"stages": [
|
||||
{"stage_id": 0, "gpu_memory_utilization": 0.9},
|
||||
{"stage_id": 1, "gpu_memory_utilization": 0.4},
|
||||
{"stage_id": 2, "gpu_memory_utilization": 0.5, "devices": "2"},
|
||||
],
|
||||
},
|
||||
"xpu": {
|
||||
"stages": [
|
||||
{
|
||||
@@ -354,23 +347,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
|
||||
},
|
||||
],
|
||||
"platforms": {
|
||||
"rocm": {
|
||||
"stages": [
|
||||
{"stage_id": 0, "max_num_seqs": 1, "default_sampling_params": {"max_tokens": 100}},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": True,
|
||||
"default_sampling_params": {"max_tokens": 100},
|
||||
},
|
||||
{
|
||||
"stage_id": 2,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 1000000,
|
||||
"default_sampling_params": {"max_tokens": 200},
|
||||
},
|
||||
],
|
||||
},
|
||||
"xpu": {
|
||||
"stages": [
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user