365 lines
13 KiB
Python
365 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Generate a GPU memory monitoring report (HTML with charts and simple anomaly markers)
|
||
from the CSV produced by `resource_monitor.sh`.
|
||
|
||
This is used to generate an archivable report in CI after a long-running stability
|
||
test so that the report remains available even after environment cleanup.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import os
|
||
import sys
|
||
from collections import defaultdict
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
from vllm.logger import init_logger
|
||
|
||
logger = init_logger(__name__)
|
||
|
||
|
||
def load_csv(csv_path: str) -> list[dict]:
|
||
"""Load and parse rows from a GPU monitoring CSV file.
|
||
|
||
Args:
|
||
csv_path: Path to the CSV file.
|
||
|
||
Returns:
|
||
A list of parsed row dicts. Each row contains keys such as
|
||
`timestamp_epoch`, `gpu_index`, and `memory_*`.
|
||
Invalid rows are skipped and logged.
|
||
"""
|
||
rows = []
|
||
with open(csv_path, newline="", encoding="utf-8") as f:
|
||
for r in csv.DictReader(f):
|
||
try:
|
||
r["timestamp_epoch"] = int(float(r["timestamp_epoch"]))
|
||
r["gpu_index"] = int(r["gpu_index"])
|
||
r["memory_used_mb"] = int(r["memory_used_mb"])
|
||
r["memory_total_mb"] = int(r["memory_total_mb"])
|
||
r["memory_util_pct"] = int(r["memory_util_pct"])
|
||
rows.append(r)
|
||
except (KeyError, ValueError) as e:
|
||
logger.debug("Skip invalid CSV row: %s", e)
|
||
continue
|
||
return rows
|
||
|
||
|
||
def compute_stats(rows: list[dict]) -> dict:
|
||
"""Compute per-GPU min/max/avg/P50/P95 memory utilization and sample counts.
|
||
|
||
Args:
|
||
rows: Rows returned by `load_csv`.
|
||
|
||
Returns:
|
||
A stats dict keyed by `gpu_index`, with values containing
|
||
`min`/`max`/`avg`/`p50`/`p95`/`samples`.
|
||
"""
|
||
by_gpu = defaultdict(list)
|
||
for r in rows:
|
||
by_gpu[r["gpu_index"]].append(r["memory_util_pct"])
|
||
stats = {}
|
||
for gpu, pcts in by_gpu.items():
|
||
if not pcts:
|
||
continue
|
||
pcts_sorted = sorted(pcts)
|
||
n = len(pcts_sorted)
|
||
stats[gpu] = {
|
||
"min": min(pcts),
|
||
"max": max(pcts),
|
||
"avg": round(sum(pcts) / n, 1),
|
||
"p50": pcts_sorted[n // 2] if n else 0,
|
||
"p95": pcts_sorted[int(n * 0.95)] if n > 1 else pcts_sorted[0],
|
||
"samples": n,
|
||
}
|
||
return stats
|
||
|
||
|
||
def find_anomalies(rows: list[dict], high_pct: int = 95, low_pct: int = 5) -> list[dict]:
|
||
"""Find simple anomalies where memory utilization exceeds `high_pct` or drops below `low_pct`."""
|
||
anomalies = []
|
||
for r in rows:
|
||
pct = r["memory_util_pct"]
|
||
ts_iso = r.get("timestamp_iso") or datetime.fromtimestamp(r["timestamp_epoch"]).strftime("%Y-%m-%d %H:%M:%S")
|
||
extra = {"timestamp_iso": ts_iso}
|
||
if pct >= high_pct:
|
||
anomalies.append({**r, **extra, "type": "high", "threshold": high_pct})
|
||
elif pct <= low_pct:
|
||
anomalies.append({**r, **extra, "type": "low", "threshold": low_pct})
|
||
return anomalies
|
||
|
||
|
||
def _anomaly_sort_key(a: dict) -> tuple[int, str, int]:
|
||
return (a["gpu_index"], a["type"], a["timestamp_epoch"])
|
||
|
||
|
||
def merge_anomalies_into_periods(anomalies: list[dict], max_gap_seconds: int = 120) -> list[dict]:
|
||
"""Merge consecutive anomalies (same GPU, same type) into time periods for display.
|
||
|
||
This avoids truncation: instead of showing only the first N raw points, we show
|
||
each continuous period once, so all below-threshold (or above-threshold) periods
|
||
are visible in the report.
|
||
"""
|
||
if not anomalies:
|
||
return []
|
||
# Sort by GPU, type, then time so we can merge consecutive same-GPU same-type.
|
||
sorted_anomalies = sorted(anomalies, key=_anomaly_sort_key)
|
||
periods = []
|
||
for a in sorted_anomalies:
|
||
ts = a["timestamp_epoch"]
|
||
ts_iso = a.get("timestamp_iso") or datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||
if not periods:
|
||
periods.append(
|
||
{
|
||
"gpu_index": a["gpu_index"],
|
||
"type": a["type"],
|
||
"threshold": a["threshold"],
|
||
"start_epoch": ts,
|
||
"end_epoch": ts,
|
||
"start_iso": ts_iso,
|
||
"end_iso": ts_iso,
|
||
"min_pct": a["memory_util_pct"],
|
||
"max_pct": a["memory_util_pct"],
|
||
"samples": 1,
|
||
}
|
||
)
|
||
continue
|
||
last = periods[-1]
|
||
if (
|
||
last["gpu_index"] == a["gpu_index"]
|
||
and last["type"] == a["type"]
|
||
and (ts - last["end_epoch"]) <= max_gap_seconds
|
||
):
|
||
last["end_epoch"] = ts
|
||
last["end_iso"] = ts_iso
|
||
last["min_pct"] = min(last["min_pct"], a["memory_util_pct"])
|
||
last["max_pct"] = max(last["max_pct"], a["memory_util_pct"])
|
||
last["samples"] += 1
|
||
else:
|
||
periods.append(
|
||
{
|
||
"gpu_index": a["gpu_index"],
|
||
"type": a["type"],
|
||
"threshold": a["threshold"],
|
||
"start_epoch": ts,
|
||
"end_epoch": ts,
|
||
"start_iso": ts_iso,
|
||
"end_iso": ts_iso,
|
||
"min_pct": a["memory_util_pct"],
|
||
"max_pct": a["memory_util_pct"],
|
||
"samples": 1,
|
||
}
|
||
)
|
||
return periods
|
||
|
||
|
||
def build_series_by_gpu(rows: list[dict]) -> tuple[list[float], dict[int, list[float]]]:
|
||
"""Deduplicate timestamps in time order and build a memory usage series (GB) for each GPU."""
|
||
times = []
|
||
by_ts_gpu = defaultdict(dict)
|
||
for r in rows:
|
||
t = r["timestamp_epoch"]
|
||
g = r["gpu_index"]
|
||
# Convert MB to GB for chart display.
|
||
by_ts_gpu[t][g] = r["memory_used_mb"] / 1024.0
|
||
for t in sorted(by_ts_gpu.keys()):
|
||
times.append(t)
|
||
gpu_series = defaultdict(list)
|
||
for t in times:
|
||
gpus = by_ts_gpu[t]
|
||
for g in sorted(gpus.keys()):
|
||
gpu_series[g].append(gpus[g])
|
||
return times, dict(gpu_series)
|
||
|
||
|
||
def render_html(
|
||
run_id: str,
|
||
csv_path: str,
|
||
rows: list[dict],
|
||
stats: dict,
|
||
anomalies: list[dict],
|
||
out_path: str,
|
||
) -> None:
|
||
"""Generate a single-file HTML report and write it to `out_path`.
|
||
|
||
Args:
|
||
run_id: Run identifier used in the title.
|
||
csv_path: Source CSV path, used only for displaying the file name.
|
||
rows: Raw data rows.
|
||
stats: Statistics returned by `compute_stats`.
|
||
anomalies: Anomaly list returned by `find_anomalies`.
|
||
out_path: Output HTML path; parent directories are created if needed.
|
||
"""
|
||
times, gpu_series = build_series_by_gpu(rows)
|
||
# X-axis time, e.g. 02-27 11:38:07 (local timezone), which is easier to read for long stability runs.
|
||
labels_js = [f'"{datetime.fromtimestamp(t).strftime("%m-%d %H:%M:%S")}"' for t in times]
|
||
datasets_js = []
|
||
colors = ["#e94560", "#0f3460", "#533483", "#16c79a"]
|
||
for i, (gpu, series) in enumerate(sorted(gpu_series.items())):
|
||
color = colors[i % len(colors)]
|
||
data_str = ",".join(str(v) for v in series)
|
||
datasets_js.append(
|
||
f'{{ label: "GPU {gpu}", data: [{data_str}], borderColor: "{color}", '
|
||
f'backgroundColor: "{color}20", fill: true, tension: 0.2 }}'
|
||
)
|
||
|
||
stats_rows = []
|
||
for gpu in sorted(stats.keys()):
|
||
s = stats[gpu]
|
||
stats_rows.append(
|
||
f"<tr><td>GPU {gpu}</td><td>{s['min']}%</td><td>{s['max']}%</td>"
|
||
f"<td>{s['avg']}%</td><td>{s['p50']}</td><td>{s['p95']}</td><td>{s['samples']}</td></tr>"
|
||
)
|
||
stats_table = "\n".join(stats_rows)
|
||
|
||
# Use merged periods so every below/above-threshold time period is shown (no truncation).
|
||
periods = merge_anomalies_into_periods(anomalies)
|
||
anomaly_cells = []
|
||
for p in periods:
|
||
time_range = f"{p['start_iso']} — {p['end_iso']}" if p["start_iso"] != p["end_iso"] else p["start_iso"]
|
||
pct_str = f"{p['min_pct']}–{p['max_pct']}%" if p["min_pct"] != p["max_pct"] else f"{p['min_pct']}%"
|
||
anomaly_cells.append(
|
||
f"<tr><td>{time_range}</td><td>GPU {p['gpu_index']}</td>"
|
||
f"<td>{pct_str}</td><td>{p['type']} (threshold {p['threshold']}%)</td><td>{p['samples']}</td></tr>"
|
||
)
|
||
anomaly_table = "\n".join(anomaly_cells) if anomaly_cells else "<tr><td colspan='5'>None</td></tr>"
|
||
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<title>GPU memory monitor report - {run_id}</title>
|
||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
|
||
<style>
|
||
body {{ font-family: system-ui,sans-serif; margin: 1rem; background: #1a1a2e; color: #eee; }}
|
||
h1 {{ font-size: 1.25rem; }} .meta {{ color: #888; margin-bottom: 1rem; }}
|
||
table {{ border-collapse: collapse; margin: 1rem 0; }} th, td {{ border: 1px solid #333; padding: 0.4rem 0.6rem; text-align: left; }}
|
||
th {{ background: #16213e; }} canvas {{ max-height: 400px; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Stability GPU memory monitor report</h1>
|
||
<p class="meta">Run: {run_id} | Data file: {os.path.basename(csv_path)} | Samples: {len(rows)}</p>
|
||
<h2>Statistics</h2>
|
||
<table>
|
||
<tr><th>GPU</th><th>Min %</th><th>Max %</th><th>Avg %</th><th>P50</th><th>P95</th><th>Samples</th></tr>
|
||
{stats_table}
|
||
</table>
|
||
<h2>Memory utilization over time</h2>
|
||
<button id="toggleMode">Split per GPU</button>
|
||
<div id="chartsContainer">
|
||
<canvas id="chart"></canvas>
|
||
</div>
|
||
<h2>Anomalies (high/low threshold)</h2>
|
||
<p class="meta">Consecutive samples merged into periods; all periods listed.</p>
|
||
<table>
|
||
<tr><th>Period</th><th>GPU</th><th>Util %</th><th>Type</th><th>Samples</th></tr>
|
||
{anomaly_table}
|
||
</table>
|
||
<script>
|
||
const labels = [{",".join(labels_js)}];
|
||
const gpuDatasets = [{",".join(datasets_js)}];
|
||
|
||
let mode = "combined";
|
||
let charts = [];
|
||
|
||
const commonOptions = {{
|
||
responsive: true,
|
||
scales: {{
|
||
y: {{
|
||
title: {{
|
||
display: true,
|
||
text: "Memory used (GB)"
|
||
}}
|
||
}}
|
||
}}
|
||
}};
|
||
|
||
function destroyCharts() {{
|
||
charts.forEach(c => c.destroy());
|
||
charts = [];
|
||
}}
|
||
|
||
function renderCombined() {{
|
||
const container = document.getElementById("chartsContainer");
|
||
container.innerHTML = '<canvas id="chart"></canvas>';
|
||
const ctx = document.getElementById("chart");
|
||
charts = [new Chart(ctx, {{
|
||
type: "line",
|
||
data: {{ labels, datasets: gpuDatasets }},
|
||
options: commonOptions
|
||
}})];
|
||
}}
|
||
|
||
function renderSplit() {{
|
||
const container = document.getElementById("chartsContainer");
|
||
container.innerHTML = "";
|
||
charts = [];
|
||
gpuDatasets.forEach((ds, idx) => {{
|
||
const canvas = document.createElement("canvas");
|
||
canvas.id = "chart-gpu-" + idx;
|
||
container.appendChild(canvas);
|
||
charts.push(new Chart(canvas, {{
|
||
type: "line",
|
||
data: {{ labels, datasets: [ds] }},
|
||
options: commonOptions
|
||
}}));
|
||
}});
|
||
}}
|
||
|
||
document.getElementById("toggleMode").addEventListener("click", () => {{
|
||
destroyCharts();
|
||
if (mode === "combined") {{
|
||
mode = "split";
|
||
document.getElementById("toggleMode").textContent = "Combine all GPUs";
|
||
renderSplit();
|
||
}} else {{
|
||
mode = "combined";
|
||
document.getElementById("toggleMode").textContent = "Split per GPU";
|
||
renderCombined();
|
||
}}
|
||
}});
|
||
|
||
// Default: combined chart
|
||
renderCombined();
|
||
</script>
|
||
</body>
|
||
</html>
|
||
"""
|
||
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
||
with open(out_path, "w", encoding="utf-8") as f:
|
||
f.write(html)
|
||
|
||
|
||
def main() -> int:
|
||
"""Entry point: read the CSV path and optional output path from the command line and generate `report.html`.
|
||
|
||
Returns:
|
||
0 on success, 1 on invalid arguments or invalid data.
|
||
"""
|
||
if len(sys.argv) < 2:
|
||
logger.error("Usage: generate_report.py <gpu_metrics.csv> [output.html]")
|
||
return 1
|
||
csv_path = sys.argv[1]
|
||
out_path = sys.argv[2] if len(sys.argv) > 2 else csv_path.replace(".csv", "_report.html")
|
||
if not os.path.isfile(csv_path):
|
||
logger.error("File not found: %s", csv_path)
|
||
return 1
|
||
run_id = Path(csv_path).parent.name
|
||
rows = load_csv(csv_path)
|
||
if not rows:
|
||
logger.error("CSV has no valid data")
|
||
return 1
|
||
stats = compute_stats(rows)
|
||
anomalies = find_anomalies(rows)
|
||
render_html(run_id, csv_path, rows, stats, anomalies, out_path)
|
||
print(out_path)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|