[Test] L5 Long-Term Stability Test and GPU Memory Monitoring Main L5 last (#1799)

This commit is contained in:
zhumingjue138 authored and GitHub committed 2026-03-23 19:04:05 +08:00
1 parent 65d76a4127
commit c14ab819e3
15 files changed
+1613 -101

No files matched your search

+1
View File
@@ -0,0 +1 @@
# scripts package for stability benchmark
@@ -0,0 +1,364 @@
#!/usr/bin/env python3
"""
Generate a GPU memory monitoring report (HTML with charts and simple anomaly markers)
from the CSV produced by `resource_monitor.sh`.
This is used to generate an archivable report in CI after a long-running stability
test so that the report remains available even after environment cleanup.
"""
from __future__ import annotations
import csv
import os
import sys
from collections import defaultdict
from datetime import datetime
from pathlib import Path
from vllm.logger import init_logger
logger = init_logger(__name__)
def load_csv(csv_path: str) -> list[dict]:
"""Load and parse rows from a GPU monitoring CSV file.
Args:
csv_path: Path to the CSV file.
Returns:
A list of parsed row dicts. Each row contains keys such as
`timestamp_epoch`, `gpu_index`, and `memory_*`.
Invalid rows are skipped and logged.
"""
rows = []
with open(csv_path, newline="", encoding="utf-8") as f:
for r in csv.DictReader(f):
try:
r["timestamp_epoch"] = int(float(r["timestamp_epoch"]))
r["gpu_index"] = int(r["gpu_index"])
r["memory_used_mb"] = int(r["memory_used_mb"])
r["memory_total_mb"] = int(r["memory_total_mb"])
r["memory_util_pct"] = int(r["memory_util_pct"])
rows.append(r)
except (KeyError, ValueError) as e:
logger.debug("Skip invalid CSV row: %s", e)
continue
return rows
def compute_stats(rows: list[dict]) -> dict:
"""Compute per-GPU min/max/avg/P50/P95 memory utilization and sample counts.
Args:
rows: Rows returned by `load_csv`.
Returns:
A stats dict keyed by `gpu_index`, with values containing
`min`/`max`/`avg`/`p50`/`p95`/`samples`.
"""
by_gpu = defaultdict(list)
for r in rows:
by_gpu[r["gpu_index"]].append(r["memory_util_pct"])
stats = {}
for gpu, pcts in by_gpu.items():
if not pcts:
continue
pcts_sorted = sorted(pcts)
n = len(pcts_sorted)
stats[gpu] = {
"min": min(pcts),
"max": max(pcts),
"avg": round(sum(pcts) / n, 1),
"p50": pcts_sorted[n // 2] if n else 0,
"p95": pcts_sorted[int(n * 0.95)] if n > 1 else pcts_sorted[0],
"samples": n,
}
return stats
def find_anomalies(rows: list[dict], high_pct: int = 95, low_pct: int = 5) -> list[dict]:
"""Find simple anomalies where memory utilization exceeds `high_pct` or drops below `low_pct`."""
anomalies = []
for r in rows:
pct = r["memory_util_pct"]
ts_iso = r.get("timestamp_iso") or datetime.fromtimestamp(r["timestamp_epoch"]).strftime("%Y-%m-%d %H:%M:%S")
extra = {"timestamp_iso": ts_iso}
if pct >= high_pct:
anomalies.append({**r, **extra, "type": "high", "threshold": high_pct})
elif pct <= low_pct:
anomalies.append({**r, **extra, "type": "low", "threshold": low_pct})
return anomalies
def _anomaly_sort_key(a: dict) -> tuple[int, str, int]:
return (a["gpu_index"], a["type"], a["timestamp_epoch"])
def merge_anomalies_into_periods(anomalies: list[dict], max_gap_seconds: int = 120) -> list[dict]:
"""Merge consecutive anomalies (same GPU, same type) into time periods for display.
This avoids truncation: instead of showing only the first N raw points, we show
each continuous period once, so all below-threshold (or above-threshold) periods
are visible in the report.
"""
if not anomalies:
return []
# Sort by GPU, type, then time so we can merge consecutive same-GPU same-type.
sorted_anomalies = sorted(anomalies, key=_anomaly_sort_key)
periods = []
for a in sorted_anomalies:
ts = a["timestamp_epoch"]
ts_iso = a.get("timestamp_iso") or datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
if not periods:
periods.append(
{
"gpu_index": a["gpu_index"],
"type": a["type"],
"threshold": a["threshold"],
"start_epoch": ts,
"end_epoch": ts,
"start_iso": ts_iso,
"end_iso": ts_iso,
"min_pct": a["memory_util_pct"],
"max_pct": a["memory_util_pct"],
"samples": 1,
}
)
continue
last = periods[-1]
if (
last["gpu_index"] == a["gpu_index"]
and last["type"] == a["type"]
and (ts - last["end_epoch"]) <= max_gap_seconds
):
last["end_epoch"] = ts
last["end_iso"] = ts_iso
last["min_pct"] = min(last["min_pct"], a["memory_util_pct"])
last["max_pct"] = max(last["max_pct"], a["memory_util_pct"])
last["samples"] += 1
else:
periods.append(
{
"gpu_index": a["gpu_index"],
"type": a["type"],
"threshold": a["threshold"],
"start_epoch": ts,
"end_epoch": ts,
"start_iso": ts_iso,
"end_iso": ts_iso,
"min_pct": a["memory_util_pct"],
"max_pct": a["memory_util_pct"],
"samples": 1,
}
)
return periods
def build_series_by_gpu(rows: list[dict]) -> tuple[list[float], dict[int, list[float]]]:
"""Deduplicate timestamps in time order and build a memory usage series (GB) for each GPU."""
times = []
by_ts_gpu = defaultdict(dict)
for r in rows:
t = r["timestamp_epoch"]
g = r["gpu_index"]
# Convert MB to GB for chart display.
by_ts_gpu[t][g] = r["memory_used_mb"] / 1024.0
for t in sorted(by_ts_gpu.keys()):
times.append(t)
gpu_series = defaultdict(list)
for t in times:
gpus = by_ts_gpu[t]
for g in sorted(gpus.keys()):
gpu_series[g].append(gpus[g])
return times, dict(gpu_series)
def render_html(
run_id: str,
csv_path: str,
rows: list[dict],
stats: dict,
anomalies: list[dict],
out_path: str,
) -> None:
"""Generate a single-file HTML report and write it to `out_path`.
Args:
run_id: Run identifier used in the title.
csv_path: Source CSV path, used only for displaying the file name.
rows: Raw data rows.
stats: Statistics returned by `compute_stats`.
anomalies: Anomaly list returned by `find_anomalies`.
out_path: Output HTML path; parent directories are created if needed.
"""
times, gpu_series = build_series_by_gpu(rows)
# X-axis time, e.g. 02-27 11:38:07 (local timezone), which is easier to read for long stability runs.
labels_js = [f'"{datetime.fromtimestamp(t).strftime("%m-%d %H:%M:%S")}"' for t in times]
datasets_js = []
colors = ["#e94560", "#0f3460", "#533483", "#16c79a"]
for i, (gpu, series) in enumerate(sorted(gpu_series.items())):
color = colors[i % len(colors)]
data_str = ",".join(str(v) for v in series)
datasets_js.append(
f'{{ label: "GPU {gpu}", data: [{data_str}], borderColor: "{color}", '
f'backgroundColor: "{color}20", fill: true, tension: 0.2 }}'
)
stats_rows = []
for gpu in sorted(stats.keys()):
s = stats[gpu]
stats_rows.append(
f"<tr><td>GPU {gpu}</td><td>{s['min']}%</td><td>{s['max']}%</td>"
f"<td>{s['avg']}%</td><td>{s['p50']}</td><td>{s['p95']}</td><td>{s['samples']}</td></tr>"
)
stats_table = "\n".join(stats_rows)
# Use merged periods so every below/above-threshold time period is shown (no truncation).
periods = merge_anomalies_into_periods(anomalies)
anomaly_cells = []
for p in periods:
time_range = f"{p['start_iso']} — {p['end_iso']}" if p["start_iso"] != p["end_iso"] else p["start_iso"]
pct_str = f"{p['min_pct']}–{p['max_pct']}%" if p["min_pct"] != p["max_pct"] else f"{p['min_pct']}%"
anomaly_cells.append(
f"<tr><td>{time_range}</td><td>GPU {p['gpu_index']}</td>"
f"<td>{pct_str}</td><td>{p['type']} (threshold {p['threshold']}%)</td><td>{p['samples']}</td></tr>"
)
anomaly_table = "\n".join(anomaly_cells) if anomaly_cells else "<tr><td colspan='5'>None</td></tr>"
html = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>GPU memory monitor report - {run_id}</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
<style>
body {{ font-family: system-ui,sans-serif; margin: 1rem; background: #1a1a2e; color: #eee; }}
h1 {{ font-size: 1.25rem; }} .meta {{ color: #888; margin-bottom: 1rem; }}
table {{ border-collapse: collapse; margin: 1rem 0; }} th, td {{ border: 1px solid #333; padding: 0.4rem 0.6rem; text-align: left; }}
th {{ background: #16213e; }} canvas {{ max-height: 400px; }}
</style>
</head>
<body>
<h1>Stability GPU memory monitor report</h1>
<p class="meta">Run: {run_id} | Data file: {os.path.basename(csv_path)} | Samples: {len(rows)}</p>
<h2>Statistics</h2>
<table>
<tr><th>GPU</th><th>Min %</th><th>Max %</th><th>Avg %</th><th>P50</th><th>P95</th><th>Samples</th></tr>
{stats_table}
</table>
<h2>Memory utilization over time</h2>
<button id="toggleMode">Split per GPU</button>
<div id="chartsContainer">
<canvas id="chart"></canvas>
</div>
<h2>Anomalies (high/low threshold)</h2>
<p class="meta">Consecutive samples merged into periods; all periods listed.</p>
<table>
<tr><th>Period</th><th>GPU</th><th>Util %</th><th>Type</th><th>Samples</th></tr>
{anomaly_table}
</table>
<script>
const labels = [{",".join(labels_js)}];
const gpuDatasets = [{",".join(datasets_js)}];
let mode = "combined";
let charts = [];
const commonOptions = {{
responsive: true,
scales: {{
y: {{
title: {{
display: true,
text: "Memory used (GB)"
}}
}}
}}
}};
function destroyCharts() {{
charts.forEach(c => c.destroy());
charts = [];
}}
function renderCombined() {{
const container = document.getElementById("chartsContainer");
container.innerHTML = '<canvas id="chart"></canvas>';
const ctx = document.getElementById("chart");
charts = [new Chart(ctx, {{
type: "line",
data: {{ labels, datasets: gpuDatasets }},
options: commonOptions
}})];
}}
function renderSplit() {{
const container = document.getElementById("chartsContainer");
container.innerHTML = "";
charts = [];
gpuDatasets.forEach((ds, idx) => {{
const canvas = document.createElement("canvas");
canvas.id = "chart-gpu-" + idx;
container.appendChild(canvas);
charts.push(new Chart(canvas, {{
type: "line",
data: {{ labels, datasets: [ds] }},
options: commonOptions
}}));
}});
}}
document.getElementById("toggleMode").addEventListener("click", () => {{
destroyCharts();
if (mode === "combined") {{
mode = "split";
document.getElementById("toggleMode").textContent = "Combine all GPUs";
renderSplit();
}} else {{
mode = "combined";
document.getElementById("toggleMode").textContent = "Split per GPU";
renderCombined();
}}
}});
// Default: combined chart
renderCombined();
</script>
</body>
</html>
"""
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
with open(out_path, "w", encoding="utf-8") as f:
f.write(html)
def main() -> int:
"""Entry point: read the CSV path and optional output path from the command line and generate `report.html`.
Returns:
0 on success, 1 on invalid arguments or invalid data.
"""
if len(sys.argv) < 2:
logger.error("Usage: generate_report.py <gpu_metrics.csv> [output.html]")
return 1
csv_path = sys.argv[1]
out_path = sys.argv[2] if len(sys.argv) > 2 else csv_path.replace(".csv", "_report.html")
if not os.path.isfile(csv_path):
logger.error("File not found: %s", csv_path)
return 1
run_id = Path(csv_path).parent.name
rows = load_csv(csv_path)
if not rows:
logger.error("CSV has no valid data")
return 1
stats = compute_stats(rows)
anomalies = find_anomalies(rows)
render_html(run_id, csv_path, rows, stats, anomalies, out_path)
print(out_path)
return 0
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,329 @@
#!/bin/bash
#
# Stability resource monitor script (single entry point, extendable to CPU/NPU later)
# Only GPU monitoring is implemented for now; --backend reserves cpu/npu for future expansion.
#
# Subcommands: start | finalize | run -- <command>
#
# start - collect data in the background (currently only gpu: `nvidia-smi` writes CSV)
# finalize - bundle the current run (CSV + report.html) and print BUNDLE_DIR
# run - start monitoring -> execute command -> finalize (generate report.html only, no upload)
#
# Argument entry point (reserved for multiple backends):
# --backend, -b gpu | cpu | npu default: gpu; only gpu is implemented right now
#
# Environment variables:
# RESOURCE_MONITOR_DATA_ROOT data root directory (compatible with GPU_MONITOR_DATA_ROOT)
# RESOURCE_MONITOR_INTERVAL sampling interval in seconds (compatible with GPU_MONITOR_INTERVAL)
# RESOURCE_MONITOR_LOG_INTERVAL log print interval in seconds (compatible with GPU_MONITOR_LOG_INTERVAL)
# GPU_MONITOR_DEVICES [GPU backend] GPU device IDs to monitor, e.g. 0,1 or all
#
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Data root directory: placed under the stability directory (the parent of SCRIPT_DIR)
# so it stays alongside stage_configs/tests.
STABILITY_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
DATA_ROOT="${RESOURCE_MONITOR_DATA_ROOT:-${GPU_MONITOR_DATA_ROOT:-$STABILITY_DIR/gpu_monitor_data}}"
SUBCMD="${1:-}"
# Parse optional --backend|-b arguments, store the result in BACKEND,
# and keep the remaining positional arguments in the REST_ARGS array.
parse_backend_and_rest() {
BACKEND="gpu"
REST_ARGS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--backend|-b)
if [[ -z "${2:-}" ]]; then
echo "Error: --backend requires a value (gpu|cpu|npu)" >&2
exit 1
fi
BACKEND="$2"
shift 2
;;
*)
REST_ARGS+=("$1")
shift
;;
esac
done
}
# ---------- subcommand: start ----------
cmd_start() {
parse_backend_and_rest "$@"
local POS1="${REST_ARGS[0]:-${GPU_MONITOR_DEVICES:-all}}"
local POS2="${REST_ARGS[1]:-${RESOURCE_MONITOR_INTERVAL:-${GPU_MONITOR_INTERVAL:-5}}}"
case "$BACKEND" in
gpu)
start_backend_gpu "$POS1" "$POS2"
;;
cpu|npu)
echo "Error: backend '$BACKEND' not implemented yet. Currently only 'gpu' is supported." >&2
exit 1
;;
*)
echo "Error: invalid backend '$BACKEND'. Use gpu|cpu|npu." >&2
exit 1
;;
esac
}
start_backend_gpu() {
local GPU_IDS_RAW="${1:-${GPU_MONITOR_DEVICES:-all}}"
local INTERVAL="${2:-${RESOURCE_MONITOR_INTERVAL:-${GPU_MONITOR_INTERVAL:-5}}}"
[[ "$INTERVAL" =~ ^[0-9]+$ ]] && [[ "$INTERVAL" -ge 1 ]] || {
echo "Error: interval must be a positive integer (seconds)"
echo "Usage: $0 start [--backend gpu|cpu|npu] [gpu_ids] [interval_seconds]"
exit 1
}
local RUN_ID="run_$(date +%Y%m%d_%H%M%S)"
local RUN_DIR="$DATA_ROOT/$RUN_ID"
mkdir -p "$RUN_DIR"
echo "$RUN_ID" > "$DATA_ROOT/current_run_id"
local CSV_FILE="$RUN_DIR/gpu_metrics.csv"
echo "timestamp_iso,timestamp_epoch,gpu_index,memory_used_mb,memory_total_mb,memory_util_pct" > "$CSV_FILE"
local NVSMI_QUERY="index,memory.used,memory.total"
local NVSMI_IDS=""
[[ "$GPU_IDS_RAW" != "all" ]] && NVSMI_IDS="-i $GPU_IDS_RAW"
trap 'echo "[$(date +%H:%M:%S)] Stopping; data saved to $RUN_DIR"; exit 0' SIGTERM SIGINT
echo "========================================"
echo "Stability resource monitor (backend=gpu) started"
echo "RUN_ID: $RUN_ID"
echo "Data dir: $RUN_DIR"
echo "Interval: ${INTERVAL}s | GPU: $GPU_IDS_RAW"
echo "========================================"
while true; do
local TS_ISO TS_EPOCH RAW
TS_ISO=$(date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S%z')
TS_EPOCH=$(date +%s)
RAW=$(nvidia-smi --query-gpu="$NVSMI_QUERY" --format=csv,noheader,nounits $NVSMI_IDS 2>/dev/null) || true
if [[ -z "$RAW" ]]; then
sleep "$INTERVAL"
continue
fi
while IFS= read -r line; do
line=$(echo "$line" | tr -d ' ')
[[ -z "$line" ]] && continue
local idx used total pct
idx=$(echo "$line" | cut -d',' -f1)
used=$(echo "$line" | cut -d',' -f2)
total=$(echo "$line" | cut -d',' -f3)
used=${used:-0}
total=${total:-1}
[[ "$total" -le 0 ]] && total=1
pct=$((used * 100 / total))
echo "${TS_ISO},${TS_EPOCH},${idx},${used},${total},${pct}" >> "$CSV_FILE"
done <<< "$RAW"
sleep "$INTERVAL"
done
}
# ---------- subcommand: finalize ----------
cmd_finalize() {
parse_backend_and_rest "$@"
local RUN_ID="${REST_ARGS[0]:-}"
case "$BACKEND" in
gpu)
finalize_backend_gpu "$RUN_ID"
;;
cpu|npu)
echo "Error: backend '$BACKEND' not implemented yet. Currently only 'gpu' is supported." >&2
exit 1
;;
*)
echo "Error: invalid backend '$BACKEND'. Use gpu|cpu|npu." >&2
exit 1
;;
esac
}
finalize_backend_gpu() {
local RUN_ID="${1:-}"
if [[ -z "$RUN_ID" ]]; then
if [[ -f "$DATA_ROOT/current_run_id" ]]; then
RUN_ID=$(cat "$DATA_ROOT/current_run_id")
else
echo "Error: run_id not specified and $DATA_ROOT/current_run_id does not exist" >&2
exit 1
fi
fi
local RUN_DIR="$DATA_ROOT/$RUN_ID"
if [[ ! -d "$RUN_DIR" ]]; then
echo "Error: run dir does not exist: $RUN_DIR" >&2
exit 1
fi
local CSV_FILE="$RUN_DIR/gpu_metrics.csv"
if [[ ! -f "$CSV_FILE" ]]; then
echo "Error: CSV not found: $CSV_FILE" >&2
exit 1
fi
local BUNDLE_DIR="$DATA_ROOT/gpu_monitor_bundle_${RUN_ID}"
rm -rf "$BUNDLE_DIR"
mkdir -p "$BUNDLE_DIR"
cp "$CSV_FILE" "$BUNDLE_DIR/gpu_metrics.csv"
local REPORT_HTML="$BUNDLE_DIR/report.html"
if command -v python3 &>/dev/null; then
if python3 "$SCRIPT_DIR/generate_report.py" "$CSV_FILE" "$REPORT_HTML"; then
echo "Report generated: $REPORT_HTML"
else
echo "Warning: report generation failed; only CSV archived" >&2
fi
else
echo "Warning: python3 not found; skipping report" >&2
fi
cat > "$BUNDLE_DIR/README.txt" << EOF
Stability resource monitor (gpu) bundle - ${RUN_ID}
- gpu_metrics.csv: raw samples
- report.html: report with charts (open in browser to view)
EOF
local BUNDLE_ABS
BUNDLE_ABS=$(cd "$BUNDLE_DIR" && pwd)
echo "GPU_MONITOR_BUNDLE_DIR=$BUNDLE_ABS"
echo "RESOURCE_MONITOR_BUNDLE_DIR=$BUNDLE_ABS"
echo "Archive path: $BUNDLE_ABS"
}
# ---------- subcommand: run ----------
cmd_run() {
shift
local BACKEND="gpu"
local CMD=()
while [[ $# -gt 0 ]]; do
case "$1" in
--backend|-b)
if [[ -n "${2:-}" ]]; then
BACKEND="$2"
shift 2
else
shift
fi
;;
--)
shift
CMD=("$@")
break
;;
*)
shift
;;
esac
done
if [[ ${#CMD[@]} -eq 0 ]]; then
echo "Usage: $0 run [--backend gpu|cpu|npu] -- <command to run>" >&2
exit 1
fi
case "$BACKEND" in
gpu)
run_backend_gpu "${CMD[@]}"
;;
cpu|npu)
echo "Error: backend '$BACKEND' not implemented yet. Currently only 'gpu' is supported." >&2
exit 1
;;
*)
echo "Error: invalid backend '$BACKEND'. Use gpu|cpu|npu." >&2
exit 1
;;
esac
}
run_backend_gpu() {
local CMD=("$@")
export RESOURCE_MONITOR_DATA_ROOT="${RESOURCE_MONITOR_DATA_ROOT:-${GPU_MONITOR_DATA_ROOT:-$STABILITY_DIR/gpu_monitor_data}}"
export GPU_MONITOR_DATA_ROOT="${GPU_MONITOR_DATA_ROOT:-$RESOURCE_MONITOR_DATA_ROOT}"
export RESOURCE_MONITOR_INTERVAL="${RESOURCE_MONITOR_INTERVAL:-${GPU_MONITOR_INTERVAL:-5}}"
export GPU_MONITOR_INTERVAL="${GPU_MONITOR_INTERVAL:-${GPU_MONITOR_INTERVAL:-5}}"
export GPU_MONITOR_DEVICES="${GPU_MONITOR_DEVICES:-all}"
export RESOURCE_MONITOR_LOG_INTERVAL="${RESOURCE_MONITOR_LOG_INTERVAL:-${GPU_MONITOR_LOG_INTERVAL:-15}}"
export GPU_MONITOR_LOG_INTERVAL="${RESOURCE_MONITOR_LOG_INTERVAL:-${GPU_MONITOR_LOG_INTERVAL:-15}}"
local REPO_ROOT
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
local MONITOR_PID="" LOG_REPORTER_PID=""
local TEST_EXIT_CODE=0
cleanup() {
[[ -n "$LOG_REPORTER_PID" ]] && kill -0 "$LOG_REPORTER_PID" 2>/dev/null && kill "$LOG_REPORTER_PID" 2>/dev/null || true
[[ -n "$MONITOR_PID" ]] && kill -0 "$MONITOR_PID" 2>/dev/null && kill "$MONITOR_PID" 2>/dev/null || true
if [[ -f "$GPU_MONITOR_DATA_ROOT/current_run_id" ]]; then
echo "--- Finalizing: bundling resource (gpu) monitor data ---"
local TMPF BUNDLE_LINE
TMPF=$(mktemp)
"$SCRIPT_DIR/resource_monitor.sh" finalize --backend gpu 2>&1 | tee "$TMPF"
BUNDLE_LINE=$(grep '^GPU_MONITOR_BUNDLE_DIR=' "$TMPF" || true)
rm -f "$TMPF"
if [[ -n "$BUNDLE_LINE" ]]; then
eval "$BUNDLE_LINE"
if [[ -d "${GPU_MONITOR_BUNDLE_DIR:-}" ]]; then
echo "--- Resource monitor bundle dir: $GPU_MONITOR_BUNDLE_DIR ---"
echo "--- Line chart: open in browser: $GPU_MONITOR_BUNDLE_DIR/report.html ---"
fi
fi
fi
exit "${TEST_EXIT_CODE:-0}"
}
trap cleanup EXIT
if command -v nvidia-smi &>/dev/null; then
"$SCRIPT_DIR/resource_monitor.sh" start --backend gpu "$GPU_MONITOR_DEVICES" "$RESOURCE_MONITOR_INTERVAL" &
MONITOR_PID=$!
echo "[Resource Monitor (gpu)] Started (PID $MONITOR_PID), interval=${RESOURCE_MONITOR_INTERVAL}s, devices=$GPU_MONITOR_DEVICES; log every ${RESOURCE_MONITOR_LOG_INTERVAL}s."
else
echo "[Resource Monitor (gpu)] nvidia-smi not found; skipping."
fi
(
sleep 10
while true; do
sleep "$RESOURCE_MONITOR_LOG_INTERVAL"
local RID_FILE RUN_ID CSV LINE
RID_FILE="$GPU_MONITOR_DATA_ROOT/current_run_id"
[[ -f "$RID_FILE" ]] || continue
RUN_ID=$(cat "$RID_FILE" 2>/dev/null)
CSV="$GPU_MONITOR_DATA_ROOT/$RUN_ID/gpu_metrics.csv"
[[ -f "$CSV" ]] || continue
LINE=$(tail -1 "$CSV" 2>/dev/null)
[[ -n "$LINE" ]] && echo "[GPU] $LINE"
done
) &
LOG_REPORTER_PID=$!
(cd "$REPO_ROOT" && "${CMD[@]}") || TEST_EXIT_CODE=$?
exit $TEST_EXIT_CODE
}
# ---------- dispatch ----------
case "$SUBCMD" in
start) cmd_start "${@:2}" ;;
finalize) cmd_finalize "${@:2}" ;;
run) cmd_run "$@" ;;
*)
echo "Usage: $0 { start [--backend gpu|cpu|npu] [gpu_ids] [interval] | finalize [--backend gpu|cpu|npu] [run_id] | run [--backend gpu|cpu|npu] -- <command> }" >&2
echo " start - background monitor (currently only backend=gpu)" >&2
echo " finalize - bundle current run, print GPU_MONITOR_BUNDLE_DIR= / RESOURCE_MONITOR_BUNDLE_DIR=" >&2
echo " run - start + command + finalize (generate report.html only)" >&2
echo " --backend gpu|cpu|npu (default: gpu; cpu/npu reserved for future use)" >&2
exit 1
;;
esac
@@ -0,0 +1,286 @@
"""
Stability test cases: start OmniServer first, then run benchmark traffic with either
`request-rate` or `max-concurrency` for a fixed duration. No new requests are sent
after the duration is reached, and the test asserts that there are no failed requests.
The overall flow matches the perf logic: `load_configs`, `modify_stage`,
`create_unique_server_params`, `create_test_parameter_mapping`,
`get_benchmark_params_for_server`, `create_benchmark_indices`, and the
`omni_server` fixture are aligned with perf. Only the benchmark execution
(`run_stability_benchmark`, which is duration-based here) and the test cases differ.
All test-specific parameters, such as `duration_sec`, `request_rate` /
`max_concurrency`, and `num_prompts_per_batch`, are configured in
`tests/dfx/stability/tests/test.json` and are no longer overridden
through environment variables.
"""
import json
import os
import threading
import time
from pathlib import Path
from typing import Any
import pytest
from tests.conftest import OmniServer
from tests.dfx.conftest import (
create_benchmark_indices,
create_test_parameter_mapping,
create_unique_server_params,
get_benchmark_params_for_server,
load_configs,
)
from tests.dfx.perf.scripts.run_benchmark import run_benchmark
STABILITY_DIR = Path(__file__).resolve().parent.parent
STAGE_CONFIGS_DIR = STABILITY_DIR / "stage_configs"
CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test.json")
DEFAULT_NUM_PROMPTS_PER_BATCH = 20
try:
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
except FileNotFoundError:
BENCHMARK_CONFIGS = []
test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
_omni_server_lock = threading.Lock()
benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
def _build_base_args(params: dict[str, Any], host: str, port: int) -> list[str]:
exclude = {
"request_rate",
"max_concurrency",
"num_prompts",
"baseline",
"duration_sec",
"num_prompts_per_batch",
}
args = ["--host", host, "--port", str(port)]
for key, value in params.items():
if key in exclude or value is None:
continue
arg_name = f"--{key.replace('_', '-')}"
if isinstance(value, bool) and value:
args.append(arg_name)
elif isinstance(value, dict):
args.extend([arg_name, json.dumps(value, ensure_ascii=False, separators=(",", ":"))])
elif not isinstance(value, bool):
args.extend([arg_name, str(value)])
return args
def _run_one_benchmark_batch(
host: str,
port: int,
params: dict[str, Any],
num_prompts: int,
request_rate: float | None,
max_concurrency: int | None,
result_dir: str,
batch_index: int,
) -> dict[str, Any]:
base = _build_base_args(params, host, port)
if request_rate is not None:
args = base + ["--request-rate", str(request_rate), "--num-prompts", str(num_prompts)]
flow = request_rate
else:
args = base + [
"--max-concurrency",
str(max_concurrency),
"--num-prompts",
str(num_prompts),
"--request-rate",
"inf",
]
flow = max_concurrency
dataset_name = params.get("dataset_name", "random")
old_benchmark_dir = os.environ.get("BENCHMARK_DIR")
try:
os.environ["BENCHMARK_DIR"] = result_dir
result = run_benchmark(
args=args,
test_name="stability",
flow=flow,
dataset_name=dataset_name,
num_prompt=num_prompts,
)
return result
except (FileNotFoundError, OSError) as e:
# Surface batch failure so the stability test does not false-pass when
# run_benchmark fails before writing JSON (e.g. command not found).
return {
"completed": 0,
"failed": 1,
"duration": 0.0,
"errors": [f"Benchmark batch failed: {type(e).__name__}: {e}"],
}
finally:
if old_benchmark_dir is not None:
os.environ["BENCHMARK_DIR"] = old_benchmark_dir
elif "BENCHMARK_DIR" in os.environ:
os.environ.pop("BENCHMARK_DIR")
def _merge_batch_results(batch_results: list[dict[str, Any]], total_duration_sec: float) -> dict[str, Any]:
if not batch_results:
return {"completed": 0, "failed": 0, "duration": total_duration_sec, "errors": []}
completed = sum(r.get("completed", 0) for r in batch_results)
failed = sum(r.get("failed", 0) for r in batch_results)
merged: dict[str, Any] = {
"completed": completed,
"failed": failed,
"duration": total_duration_sec,
"errors": [],
}
for r in batch_results:
merged["errors"].extend(r.get("errors") or [])
return merged
def _print_merged_report(result: dict[str, Any]) -> None:
"""Print the final summary: successful requests, failed requests, and total duration only."""
fmt = "{:<40} {:<10}"
fmt_float = "{:<40} {:<10.2f}"
completed = result.get("completed", 0)
failed = result.get("failed", 0)
duration = float(result.get("duration", 0.0) or 0.0)
print("\n============ Stability Benchmark Summary ============")
print(fmt.format("Successful requests:", completed))
print(fmt.format("Failed requests:", failed))
print(fmt_float.format("Total duration (s):", duration))
print("==================================================\n")
def run_stability_benchmark(
host: str,
port: int,
duration_sec: int | float,
params: dict[str, Any],
*,
request_rate: float | None = None,
max_concurrency: int | None = None,
result_filename: str | None = None,
result_dir: str = "./",
num_prompts_per_batch: int = DEFAULT_NUM_PROMPTS_PER_BATCH,
) -> dict[str, Any]:
if (request_rate is None) == (max_concurrency is None):
raise ValueError("Exactly one of request_rate or max_concurrency must be specified")
start_time = time.perf_counter()
batch_results: list[dict[str, Any]] = []
batch_index = 0
while True:
if (time.perf_counter() - start_time) >= duration_sec:
break
result = _run_one_benchmark_batch(
host=host,
port=port,
params=params,
num_prompts=num_prompts_per_batch,
request_rate=request_rate,
max_concurrency=max_concurrency,
result_dir=result_dir,
batch_index=batch_index,
)
batch_results.append(result)
batch_index += 1
if (time.perf_counter() - start_time) >= duration_sec:
break
total_duration = time.perf_counter() - start_time
merged = _merge_batch_results(batch_results, total_duration)
_print_merged_report(merged)
if result_filename and result_dir:
result_path = Path(result_dir) / result_filename
with open(result_path, "w", encoding="utf-8") as f:
json.dump(merged, f, indent=2, ensure_ascii=False)
return merged
@pytest.fixture(scope="module")
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
test_name, model, stage_config_path = request.param
print(f"Starting OmniServer with test: {test_name}, model: {model}")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
server.test_name = test_name
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
print("OmniServer stopped")
@pytest.fixture(params=benchmark_indices)
def stability_benchmark_params(request, omni_server):
"""Benchmark parameters fixture with proper parametrization (same as perf)."""
test_name, param_index = request.param
if test_name != omni_server.test_name:
pytest.skip(f"Skipping parameter for {test_name} - current server is {omni_server.test_name}")
all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
if not all_params:
raise ValueError(f"No benchmark parameters found for test: {test_name}")
if param_index >= len(all_params):
raise ValueError(f"No benchmark parameters found for index {param_index} in test: {test_name}")
current = param_index + 1
total = len(all_params)
print(f"\n Running benchmark {current}/{total} for {test_name}")
return {"test_name": test_name, "params": all_params[param_index]}
@pytest.mark.slow
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
def test_benchmark_stability(omni_server, stability_benchmark_params):
"""Run the benchmark for a fixed duration using request-rate or max-concurrency and assert zero failed requests."""
test_name = stability_benchmark_params["test_name"]
params = stability_benchmark_params["params"]
duration_sec = params.get("duration_sec", 300)
num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
request_rate = params.get("request_rate")
max_concurrency = params.get("max_concurrency")
bench_params = {
k: v
for k, v in params.items()
if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
}
result = run_stability_benchmark(
host=omni_server.host,
port=omni_server.port,
duration_sec=duration_sec,
params=bench_params,
request_rate=request_rate,
max_concurrency=max_concurrency,
result_dir=str(STABILITY_DIR),
num_prompts_per_batch=num_prompts_per_batch,
)
assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"