634 lines
26 KiB
Python
634 lines
26 KiB
Python
"""Run benchmark arms so the numbers survive a shared machine.
|
|||
|
|
|
||
|
|
Every wrong conclusion this study produced came from measurement, not from
|
||
|
|
code: a sequential grid that swung the llama.cpp control 86.7 -> 60.4 tok/s
|
||
|
|
mid-run, a 3.1 GB "saving" that was really `VmRSS` counting the mmap'd
|
||
|
|
checkpoint's pages, and a W4A8/W4A16 ordering that flipped between grids. The
|
||
|
|
discipline that fixed each of them is the same, so it lives here instead of
|
||
|
|
being re-typed into the next shell script:
|
||
|
|
|
||
|
|
* **Rotate the arm order every pass.** Drift hits arm 1 and arm 5 differently,
|
||
|
|
and interleaving alone does not fix it: if every pass runs the arms in the
|
||
|
|
same order, arm 1 is always the one measured first and carries whatever the
|
||
|
|
machine was doing at the start of each pass. Pass ``p`` starts at arm
|
||
|
|
``(p - 1) mod n``.
|
||
|
|
* **Repeat, and report the spread.** A single number from a shared host is a
|
||
|
|
claim about the host, not the code. An arm whose passes disagree by more
|
||
|
|
than `--noise-threshold` is flagged, and the summary says so.
|
||
|
|
* **Record how busy the machine was, and refuse when it is.** A pass taken
|
||
|
|
under load is kept and labelled rather than silently averaged in -- but load
|
||
|
|
average is a whole-host number, and this host has 224 cores, so it says
|
||
|
|
nothing about the 24 this benchmark is pinned to. The pre-flight reads
|
||
|
|
per-core utilisation for the *target* cores, and a host lock keeps two
|
||
|
|
benchmarks off the same cores in the first place. Two sessions benchmarking
|
||
|
|
at once wrecked several measurements here, and one grid's control drifted
|
||
|
|
30% mid-run.
|
||
|
|
* **Prefer allocation over memory.** `RssAnon` repeats to within 0.5 MB where
|
||
|
|
`VmRSS` moved 1-2 GB between identical runs.
|
||
|
|
* **Know which direction is better.** ``best`` is the maximum for a rate and
|
||
|
|
the minimum for memory, init time or latency. Reporting the maximum for every
|
||
|
|
metric published the *worst* pass as the headline memory figure.
|
||
|
|
* **Stamp provenance.** A number whose model, build and thread set were not
|
||
|
|
recorded cannot be retracted later, only doubted -- and this study has
|
||
|
|
retracted four.
|
||
|
|
* **Give an arm its own compile cache when it changes the graph.** vLLM keys
|
||
|
|
its AOT compile cache on the *config*, so two arms that differ only by an
|
||
|
|
environment variable share a cached artifact -- and if that variable changes
|
||
|
|
the traced graph, later passes load the wrong one. Three passes of a prefill
|
||
|
|
grid died with ``KeyError: 'weight_halves'`` inside an AOT-loaded forward
|
||
|
|
after the first pass saved an artifact for the other path. Set
|
||
|
|
``VLLM_CACHE_ROOT`` per arm, and note that the first pass of each arm then
|
||
|
|
pays a cold compile, which is why ``init_s`` is not comparable across passes.
|
||
|
|
* **Gate on what the host can actually resolve.** ``--check`` compares against
|
||
|
|
a checked-in expectations file, with a band per metric taken from that
|
||
|
|
metric's *observed* spread: 0.2% for ``RssAnon``, ~5% for per-op self time,
|
||
|
|
and nothing for end-to-end throughput, which swings ~30% here and cannot
|
||
|
|
carry a gate at all. A band is an argument about what is measurable, not a
|
||
|
|
tolerance someone picked.
|
||
|
|
|
||
|
|
The arms are subprocess invocations of the existing single-run benchmarks, so
|
||
|
|
the measurement code stays the one that was verified; this only decides what
|
||
|
|
to run, when, and how to read it.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import contextlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import platform
|
||
|
|
import shlex
|
||
|
|
import statistics
|
||
|
|
import subprocess
|
||
|
|
import time
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
|
||
|
|
|
||
|
|
def load_average() -> float:
|
||
|
|
return os.getloadavg()[0]
|
||
|
|
|
||
|
|
|
||
|
|
def parse_cores(spec: str) -> list[int]:
|
||
|
|
"""'0-23,48-71' -> [0..23, 48..71]"""
|
||
|
|
cores: list[int] = []
|
||
|
|
for part in spec.split(","):
|
||
|
|
part = part.strip()
|
||
|
|
if not part:
|
||
|
|
continue
|
||
|
|
if "-" in part:
|
||
|
|
lo, hi = part.split("-")
|
||
|
|
cores.extend(range(int(lo), int(hi) + 1))
|
||
|
|
else:
|
||
|
|
cores.append(int(part))
|
||
|
|
return sorted(set(cores))
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Arm:
|
||
|
|
tag: str
|
||
|
|
command: list[str]
|
||
|
|
env: dict[str, str] = field(default_factory=dict)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Observation:
|
||
|
|
tag: str
|
||
|
|
pass_no: int
|
||
|
|
metrics: dict[str, float]
|
||
|
|
load_before: float
|
||
|
|
load_after: float
|
||
|
|
wall_s: float
|
||
|
|
ok: bool
|
||
|
|
error: str = ""
|
||
|
|
|
||
|
|
@property
|
||
|
|
def busy(self) -> bool:
|
||
|
|
return max(self.load_before, self.load_after) > BUSY_LOAD
|
||
|
|
|
||
|
|
|
||
|
|
BUSY_LOAD = 8.0
|
||
|
|
"""Above this 1-minute load average a pass is labelled contended.
|
||
|
|
|
||
|
|
A poor default on its own: load average is whole-host, and on a 224-core
|
||
|
|
machine a 24-thread benchmark contributes ~24 to it by itself, so a fixed 8
|
||
|
|
labels every pass contended and the label stops meaning anything. ``main()``
|
||
|
|
derives it from the target core count instead -- our own threads plus slack --
|
||
|
|
and the per-core pre-flight is the guard that actually decides whether to run.
|
||
|
|
"""
|
||
|
|
|
||
|
|
LOWER_IS_BETTER = (
|
||
|
|
"rss", "mem", "mb", "init", "load_s", "latency", "ms", "_s", "time", "peak",
|
||
|
|
)
|
||
|
|
"""Substrings that mark a metric where smaller wins. Rates (`tg_tps`, `pp_tps`)
|
||
|
|
take the maximum; memory, init time and latency take the minimum."""
|
||
|
|
|
||
|
|
|
||
|
|
HOST_LOCK_PATH = os.environ.get("SPARK_EDGE_LOCK", "/tmp/spark_edge_bench.lock")
|
||
|
|
"""Host-level lock. It has to live somewhere every session can see, which a
|
||
|
|
session-private scratchpad by definition is not -- that is the whole point of
|
||
|
|
a host lock. Override with ``SPARK_EDGE_LOCK``."""
|
||
|
|
|
||
|
|
CORE_BUSY_PCT = 50.0
|
||
|
|
"""A target core sustained above this is a straggler. OpenMP barriers mean the
|
||
|
|
slowest thread gates the whole region, so one heavily-loaded core costs far
|
||
|
|
more than its share -- which is why this is a separate test from the aggregate
|
||
|
|
below rather than a single averaged number."""
|
||
|
|
|
||
|
|
STOLEN_PCT = 5.0
|
||
|
|
"""Refuse when this much of the target cores' aggregate capacity is going to
|
||
|
|
someone else, which bounds the bias on the measurement.
|
||
|
|
|
||
|
|
A flat per-core threshold does not work on this host. With 224 cores and
|
||
|
|
several users there is always a transient blip somewhere: a 20% rule over a
|
||
|
|
single 0.5 s window refused three consecutive launches on cores that were
|
||
|
|
idle seconds earlier, which makes the guard useless rather than strict. So
|
||
|
|
utilisation is sampled twice and the per-core minimum is taken -- a spike in
|
||
|
|
one window does not count as load -- and the decision is made on aggregate
|
||
|
|
stolen capacity plus the straggler rule."""
|
||
|
|
|
||
|
|
|
||
|
|
def _cpu_times() -> dict[int, tuple[int, int]]:
|
||
|
|
"""(busy, total) jiffies per core, from /proc/stat."""
|
||
|
|
out: dict[int, tuple[int, int]] = {}
|
||
|
|
with open("/proc/stat") as f:
|
||
|
|
for line in f:
|
||
|
|
if not line.startswith("cpu") or line.startswith("cpu "):
|
||
|
|
continue
|
||
|
|
parts = line.split()
|
||
|
|
try:
|
||
|
|
cpu = int(parts[0][3:])
|
||
|
|
except ValueError:
|
||
|
|
continue
|
||
|
|
values = [int(v) for v in parts[1:11]]
|
||
|
|
idle = values[3] + values[4] # idle + iowait
|
||
|
|
total = sum(values)
|
||
|
|
out[cpu] = (total - idle, total)
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def core_utilization(cores: list[int], window_s: float = 0.5) -> dict[int, float]:
|
||
|
|
"""Percentage busy per core over ``window_s``."""
|
||
|
|
before = _cpu_times()
|
||
|
|
time.sleep(window_s)
|
||
|
|
after = _cpu_times()
|
||
|
|
usage: dict[int, float] = {}
|
||
|
|
for cpu in cores:
|
||
|
|
if cpu not in before or cpu not in after:
|
||
|
|
continue
|
||
|
|
busy = after[cpu][0] - before[cpu][0]
|
||
|
|
total = after[cpu][1] - before[cpu][1]
|
||
|
|
usage[cpu] = 0.0 if total <= 0 else busy / total * 100.0
|
||
|
|
return usage
|
||
|
|
|
||
|
|
|
||
|
|
def preflight(cores: list[int], *, max_busy_pct: float = CORE_BUSY_PCT,
|
||
|
|
max_stolen_pct: float = STOLEN_PCT,
|
||
|
|
window_s: float = 0.75, samples: int = 2) -> dict:
|
||
|
|
"""Refuse to start when contention would bias the measurement.
|
||
|
|
|
||
|
|
Two independent windows; a core's load is the *minimum* of them, so a
|
||
|
|
transient spike is not mistaken for a busy core. Then two tests: aggregate
|
||
|
|
stolen capacity (bounds the average bias) and a per-core straggler rule
|
||
|
|
(one slow core gates every OpenMP barrier).
|
||
|
|
"""
|
||
|
|
windows = [core_utilization(cores, window_s) for _ in range(max(samples, 1))]
|
||
|
|
usage = {c: min(w.get(c, 0.0) for w in windows) for c in cores
|
||
|
|
if any(c in w for w in windows)}
|
||
|
|
stolen = (sum(usage.values()) / len(usage)) if usage else 0.0
|
||
|
|
stragglers = {c: round(u, 1) for c, u in usage.items() if u > max_busy_pct}
|
||
|
|
return {"cores": cores,
|
||
|
|
"max_busy_pct": max_busy_pct, "max_stolen_pct": max_stolen_pct,
|
||
|
|
"utilization": {c: round(u, 1) for c, u in usage.items()},
|
||
|
|
"stolen_pct": round(stolen, 2),
|
||
|
|
"busy_cores": stragglers,
|
||
|
|
"clear": stolen <= max_stolen_pct and not stragglers}
|
||
|
|
|
||
|
|
|
||
|
|
class HostLock:
|
||
|
|
"""An advisory lock so two sessions do not benchmark the same cores.
|
||
|
|
|
||
|
|
Cooperative, and says so: it excludes other runs that take it, not the
|
||
|
|
unrelated job someone starts by hand. That is what the pre-flight is for.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, path: str = HOST_LOCK_PATH, note: str = ""):
|
||
|
|
self.path, self.note, self._fd = path, note, None
|
||
|
|
|
||
|
|
def __enter__(self) -> "HostLock":
|
||
|
|
import fcntl
|
||
|
|
|
||
|
|
self._fd = open(self.path, "a+")
|
||
|
|
try:
|
||
|
|
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
|
|
except BlockingIOError:
|
||
|
|
self._fd.seek(0)
|
||
|
|
holder = self._fd.read().strip()[-200:]
|
||
|
|
self._fd.close()
|
||
|
|
self._fd = None
|
||
|
|
raise SystemExit(
|
||
|
|
f"another benchmark holds {self.path}: {holder or 'unknown'}. "
|
||
|
|
f"Wait for it, or set SPARK_EDGE_LOCK to a different path if "
|
||
|
|
f"you really are on disjoint cores."
|
||
|
|
)
|
||
|
|
self._fd.seek(0)
|
||
|
|
self._fd.truncate()
|
||
|
|
self._fd.write(f"pid={os.getpid()} {time.strftime('%H:%M:%S')} {self.note}\n")
|
||
|
|
self._fd.flush()
|
||
|
|
return self
|
||
|
|
|
||
|
|
def __exit__(self, *exc) -> None:
|
||
|
|
if self._fd is not None:
|
||
|
|
import fcntl
|
||
|
|
|
||
|
|
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
||
|
|
self._fd.close()
|
||
|
|
self._fd = None
|
||
|
|
|
||
|
|
|
||
|
|
DEFAULT_BANDS_PCT = {
|
||
|
|
"rss_anon_mb": 1.5,
|
||
|
|
"peak_rss_anon_mb": 2.0,
|
||
|
|
"rss_anon_at_context_mb": 2.0,
|
||
|
|
"init_s": 12.0,
|
||
|
|
"ready_s": 12.0,
|
||
|
|
"import_s": 12.0,
|
||
|
|
"step_ms": 6.0,
|
||
|
|
"gemm_ms": 6.0,
|
||
|
|
}
|
||
|
|
"""Regression bands, per metric, from the spread each one actually shows on
|
||
|
|
this host. ``RssAnon`` repeats to 0.2% and is given 1.5% for safety; init time
|
||
|
|
is dominated by page cache and repeats far less tightly; per-op self time
|
||
|
|
reproduces to ~5%.
|
||
|
|
|
||
|
|
Throughput is deliberately absent. ``tg_tps`` swings ~30% run to run on this
|
||
|
|
shared host, so any band wide enough not to fire constantly is too wide to
|
||
|
|
catch a real regression. Gate on memory and per-op time instead; that is the
|
||
|
|
whole reason the per-op profiler exists."""
|
||
|
|
|
||
|
|
UNGATEABLE = {"tg_tps", "pp_tps", "tps"}
|
||
|
|
"""Metrics this host cannot resolve well enough to gate on."""
|
||
|
|
|
||
|
|
|
||
|
|
def check_expectations(summaries: list["ArmSummary"], expected: dict) -> list[dict]:
|
||
|
|
"""Compare measured medians against a checked-in expectations file.
|
||
|
|
|
||
|
|
The median, not the best: `best` is one pass and a gate that fires on the
|
||
|
|
luckiest run of a noisy metric is a coin toss.
|
||
|
|
"""
|
||
|
|
bands = {**DEFAULT_BANDS_PCT, **(expected.get("bands_pct") or {})}
|
||
|
|
arms = expected.get("arms") or {}
|
||
|
|
results = []
|
||
|
|
for s in summaries:
|
||
|
|
want = (arms.get(s.tag) or {}).get(s.metric)
|
||
|
|
if want is None:
|
||
|
|
results.append({"arm": s.tag, "metric": s.metric, "status": "unlisted",
|
||
|
|
"measured": s.median, "expected": None})
|
||
|
|
continue
|
||
|
|
if s.metric in UNGATEABLE:
|
||
|
|
results.append({"arm": s.tag, "metric": s.metric, "status": "ungateable",
|
||
|
|
"measured": s.median, "expected": want,
|
||
|
|
"why": "this host cannot resolve it"})
|
||
|
|
continue
|
||
|
|
band = bands.get(s.metric, 5.0)
|
||
|
|
drift = abs(s.median - want) / want * 100.0 if want else 0.0
|
||
|
|
worse = (s.median > want) if s.lower_is_better else (s.median < want)
|
||
|
|
results.append({
|
||
|
|
"arm": s.tag, "metric": s.metric,
|
||
|
|
"measured": s.median, "expected": want,
|
||
|
|
"drift_pct": round(drift, 2), "band_pct": band,
|
||
|
|
"status": ("ok" if drift <= band else
|
||
|
|
"REGRESSION" if worse else "improved"),
|
||
|
|
})
|
||
|
|
return results
|
||
|
|
|
||
|
|
|
||
|
|
def format_check(results: list[dict]) -> str:
|
||
|
|
if not results:
|
||
|
|
return "nothing to check"
|
||
|
|
width = max(len(r["arm"]) for r in results)
|
||
|
|
lines = []
|
||
|
|
for r in results:
|
||
|
|
if r["status"] in ("unlisted", "ungateable"):
|
||
|
|
lines.append(f" {r['arm'].ljust(width)} {r['metric']:22s} "
|
||
|
|
f"{r['measured']:9.2f} {r['status']}"
|
||
|
|
+ (f" ({r['why']})" if r.get("why") else ""))
|
||
|
|
else:
|
||
|
|
lines.append(f" {r['arm'].ljust(width)} {r['metric']:22s} "
|
||
|
|
f"{r['measured']:9.2f} vs {r['expected']:9.2f} "
|
||
|
|
f"{r['drift_pct']:+5.2f}% (band {r['band_pct']:.1f}%) "
|
||
|
|
f"{r['status']}")
|
||
|
|
return "\n".join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def metric_is_lower_better(metric: str) -> bool:
|
||
|
|
name = metric.lower()
|
||
|
|
if name.endswith(("_tps", "_tok_s", "tps")):
|
||
|
|
return False
|
||
|
|
return any(tok in name for tok in LOWER_IS_BETTER)
|
||
|
|
|
||
|
|
|
||
|
|
def provenance(extra: dict | None = None) -> dict:
|
||
|
|
"""Everything needed to decide later whether a number is still valid.
|
||
|
|
|
||
|
|
Saved profiles in this tree carry no model path, context length or build,
|
||
|
|
so two of them are indistinguishable from each other and one set silently
|
||
|
|
describes a checkpoint that was later found wrong. This is the fix.
|
||
|
|
"""
|
||
|
|
root = os.environ.get("EMBEDDING_INFER_ROOT", "/data/zhoutaichang/embedding_infer")
|
||
|
|
|
||
|
|
def _sha(rel: str) -> str | None:
|
||
|
|
try:
|
||
|
|
out = subprocess.run(["git", "-C", os.path.join(root, rel),
|
||
|
|
"rev-parse", "HEAD"],
|
||
|
|
capture_output=True, text=True, timeout=10)
|
||
|
|
return out.stdout.strip() or None
|
||
|
|
except Exception:
|
||
|
|
return None
|
||
|
|
|
||
|
|
try:
|
||
|
|
affinity = sorted(os.sched_getaffinity(0))
|
||
|
|
except AttributeError:
|
||
|
|
affinity = []
|
||
|
|
return {
|
||
|
|
"recorded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||
|
|
"host": platform.node(),
|
||
|
|
"python": platform.python_version(),
|
||
|
|
"cpu_count": os.cpu_count(),
|
||
|
|
"n_affinity": len(affinity),
|
||
|
|
"affinity": affinity,
|
||
|
|
"commits": {
|
||
|
|
"embedding_infer": _sha("."),
|
||
|
|
"vllm_omni": _sha("vllm-omni"),
|
||
|
|
"vllm": _sha("vllm"),
|
||
|
|
},
|
||
|
|
"env": {k: v for k, v in sorted(os.environ.items())
|
||
|
|
if k.startswith(("OMP_", "KMP_", "MKL_", "VLLM_", "LD_PRELOAD",
|
||
|
|
"CUDA_VISIBLE_DEVICES"))},
|
||
|
|
"loadavg": os.getloadavg(),
|
||
|
|
**(extra or {}),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def run_once(arm: Arm, pass_no: int, timeout: float, keys: list[str]) -> Observation:
|
||
|
|
env = {**os.environ, **arm.env}
|
||
|
|
load_before = load_average()
|
||
|
|
t0 = time.perf_counter()
|
||
|
|
proc = subprocess.run(arm.command, env=env, capture_output=True,
|
||
|
|
text=True, timeout=timeout)
|
||
|
|
wall = time.perf_counter() - t0
|
||
|
|
load_after = load_average()
|
||
|
|
if proc.returncode != 0:
|
||
|
|
return Observation(arm.tag, pass_no, {}, load_before, load_after, wall,
|
||
|
|
ok=False, error=proc.stderr.strip()[-400:])
|
||
|
|
metrics = _scrape(proc.stdout, keys)
|
||
|
|
return Observation(arm.tag, pass_no, metrics, load_before, load_after, wall,
|
||
|
|
ok=bool(metrics))
|
||
|
|
|
||
|
|
|
||
|
|
def _scrape(stdout: str, keys: list[str]) -> dict[str, float]:
|
||
|
|
"""Pull metrics out of the last JSON object the run printed."""
|
||
|
|
start = stdout.rfind("{")
|
||
|
|
while start >= 0:
|
||
|
|
try:
|
||
|
|
payload = json.loads(stdout[start:stdout.rfind("}") + 1])
|
||
|
|
except json.JSONDecodeError:
|
||
|
|
start = stdout.rfind("{", 0, start)
|
||
|
|
continue
|
||
|
|
return {k: float(payload[k]) for k in keys
|
||
|
|
if isinstance(payload.get(k), (int, float))}
|
||
|
|
return {}
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class ArmSummary:
|
||
|
|
tag: str
|
||
|
|
metric: str
|
||
|
|
best: float
|
||
|
|
median: float
|
||
|
|
spread_pct: float
|
||
|
|
n: int
|
||
|
|
n_busy: int
|
||
|
|
lower_is_better: bool = False
|
||
|
|
noise_threshold: float = 8.0
|
||
|
|
|
||
|
|
@property
|
||
|
|
def noisy(self) -> bool:
|
||
|
|
return self.spread_pct > self.noise_threshold
|
||
|
|
|
||
|
|
|
||
|
|
NOISE_THRESHOLD = 8.0
|
||
|
|
|
||
|
|
|
||
|
|
def summarize(observations: list[Observation], metric: str,
|
||
|
|
noise_threshold: float = NOISE_THRESHOLD,
|
||
|
|
lower_is_better: bool | None = None) -> list[ArmSummary]:
|
||
|
|
"""Collapse passes into one row per arm.
|
||
|
|
|
||
|
|
``lower_is_better`` defaults to reading the metric name, so `rss_anon_mb`
|
||
|
|
and `init_s` take their minimum while `tg_tps` takes its maximum. Taking
|
||
|
|
the maximum for every metric is how the *worst* memory pass came to be
|
||
|
|
published as the best one.
|
||
|
|
"""
|
||
|
|
if lower_is_better is None:
|
||
|
|
lower_is_better = metric_is_lower_better(metric)
|
||
|
|
by_tag: dict[str, list[Observation]] = {}
|
||
|
|
for obs in observations:
|
||
|
|
if obs.ok and metric in obs.metrics:
|
||
|
|
by_tag.setdefault(obs.tag, []).append(obs)
|
||
|
|
summaries = []
|
||
|
|
for tag, rows in by_tag.items():
|
||
|
|
values = [r.metrics[metric] for r in rows]
|
||
|
|
lo, hi = min(values), max(values)
|
||
|
|
spread = 0.0 if hi == 0 else (hi - lo) / hi * 100.0
|
||
|
|
summaries.append(ArmSummary(
|
||
|
|
tag=tag, metric=metric, best=lo if lower_is_better else hi,
|
||
|
|
median=statistics.median(values), spread_pct=spread,
|
||
|
|
n=len(values), n_busy=sum(1 for r in rows if r.busy),
|
||
|
|
lower_is_better=lower_is_better, noise_threshold=noise_threshold,
|
||
|
|
))
|
||
|
|
return sorted(summaries, key=lambda s: s.best if lower_is_better else -s.best)
|
||
|
|
|
||
|
|
|
||
|
|
def indistinguishable(summaries: list[ArmSummary]) -> str | None:
|
||
|
|
"""Say so when the gap between the top two arms is inside the noise.
|
||
|
|
|
||
|
|
Per-op self time resolves ~5% on this host; end-to-end throughput resolves
|
||
|
|
~30%. An 8% difference in tg_tps is not a result, and publishing it as one
|
||
|
|
is how the W4A8/W4A16 ordering came to flip between grids.
|
||
|
|
"""
|
||
|
|
if len(summaries) < 2:
|
||
|
|
return None
|
||
|
|
first, second = summaries[0], summaries[1]
|
||
|
|
if not first.best:
|
||
|
|
return None
|
||
|
|
gap = abs(first.best - second.best) / abs(first.best) * 100.0
|
||
|
|
band = max(first.spread_pct, second.spread_pct,
|
||
|
|
DEFAULT_BANDS_PCT.get(first.metric, first.noise_threshold))
|
||
|
|
if gap > band:
|
||
|
|
return None
|
||
|
|
return (f"{first.tag} and {second.tag} differ by {gap:.1f}% on "
|
||
|
|
f"{first.metric}, inside this measurement's own spread "
|
||
|
|
f"({band:.1f}%). That is not a ranking. For a difference this "
|
||
|
|
f"size use per-op self time, which reproduces to ~5%: "
|
||
|
|
f"profile_cpu_step.py --model <build> --prompt-len <ctx>")
|
||
|
|
|
||
|
|
|
||
|
|
def format_summary(summaries: list[ArmSummary]) -> str:
|
||
|
|
if not summaries:
|
||
|
|
return "no successful observations"
|
||
|
|
width = max(len(s.tag) for s in summaries)
|
||
|
|
direction = "min" if summaries[0].lower_is_better else "max"
|
||
|
|
lines = [f"{'arm'.ljust(width)} {f'best({direction})':>9s} {'median':>9s} "
|
||
|
|
f"{'spread':>7s} n notes"]
|
||
|
|
for s in summaries:
|
||
|
|
notes = []
|
||
|
|
if s.noisy:
|
||
|
|
notes.append(f"NOISY (>{s.noise_threshold:.0f}%)")
|
||
|
|
if s.n_busy:
|
||
|
|
notes.append(f"{s.n_busy} pass(es) under load")
|
||
|
|
if s.n < 2:
|
||
|
|
notes.append("single pass — not a result")
|
||
|
|
lines.append(f"{s.tag.ljust(width)} {s.best:9.2f} {s.median:9.2f} "
|
||
|
|
f"{s.spread_pct:6.1f}% {s.n} {'; '.join(notes)}")
|
||
|
|
warning = indistinguishable(summaries)
|
||
|
|
if warning:
|
||
|
|
lines.append("")
|
||
|
|
lines.append(f" NOT A RANKING: {warning}")
|
||
|
|
return "\n".join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def run_grid(arms: list[Arm], *, passes: int, metric: str, timeout: float,
|
||
|
|
keys: list[str], settle_s: float = 0.0,
|
||
|
|
busy_load: float = BUSY_LOAD) -> list[Observation]:
|
||
|
|
"""Run every arm each pass, rotating the starting arm between passes.
|
||
|
|
|
||
|
|
Interleaving alone leaves arm 0 permanently first, so whatever the machine
|
||
|
|
does at the start of a pass always lands on the same arm. Pass ``p`` starts
|
||
|
|
at arm ``(p - 1) mod n``.
|
||
|
|
"""
|
||
|
|
globals()["BUSY_LOAD"] = busy_load
|
||
|
|
observations: list[Observation] = []
|
||
|
|
for p in range(1, passes + 1):
|
||
|
|
shift = (p - 1) % len(arms) if arms else 0
|
||
|
|
for arm in arms[shift:] + arms[:shift]:
|
||
|
|
if settle_s:
|
||
|
|
time.sleep(settle_s)
|
||
|
|
obs = run_once(arm, p, timeout, keys)
|
||
|
|
observations.append(obs)
|
||
|
|
status = "ok" if obs.ok else f"FAILED: {obs.error[:80]}"
|
||
|
|
value = obs.metrics.get(metric)
|
||
|
|
shown = f"{value:.2f}" if value is not None else "-"
|
||
|
|
flag = " [busy]" if obs.busy else ""
|
||
|
|
print(f" pass {p} {arm.tag:24s} {metric}={shown:>9s} "
|
||
|
|
f"load {obs.load_before:.1f}->{obs.load_after:.1f}{flag} {status}",
|
||
|
|
flush=True)
|
||
|
|
return observations
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> None:
|
||
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
|
|
ap.add_argument("--spec", required=True,
|
||
|
|
help="JSON file: {\"arms\": [{\"tag\":..., \"command\":[...], \"env\":{...}}]}")
|
||
|
|
ap.add_argument("--passes", type=int, default=3)
|
||
|
|
ap.add_argument("--metric", default="tg_tps")
|
||
|
|
ap.add_argument("--keys", default="tg_tps,pp_tps,init_s,rss_anon_mb,rss_mb")
|
||
|
|
ap.add_argument("--timeout", type=float, default=2400)
|
||
|
|
ap.add_argument("--settle-s", type=float, default=2.0)
|
||
|
|
ap.add_argument("--busy-load", type=float, default=None,
|
||
|
|
help="load average above which a pass is labelled "
|
||
|
|
"contended; defaults to (target cores + 8), since "
|
||
|
|
"the run itself contributes about one per core")
|
||
|
|
ap.add_argument("--noise-threshold", type=float, default=NOISE_THRESHOLD)
|
||
|
|
ap.add_argument("--out", default=None)
|
||
|
|
ap.add_argument("--cores", default="",
|
||
|
|
help="target cores, e.g. '0-23'. Checked for other "
|
||
|
|
"people's work before the run starts.")
|
||
|
|
ap.add_argument("--allow-busy", action="store_true",
|
||
|
|
help="run even if the target cores are already loaded "
|
||
|
|
"(the result is labelled, not trusted)")
|
||
|
|
ap.add_argument("--check", default=None,
|
||
|
|
help="expectations JSON; exit non-zero on a regression "
|
||
|
|
"outside the metric's measured band")
|
||
|
|
ap.add_argument("--no-lock", action="store_true",
|
||
|
|
help="skip the host lock (only if you are on cores no "
|
||
|
|
"other benchmark uses)")
|
||
|
|
a = ap.parse_args()
|
||
|
|
|
||
|
|
spec = json.loads(open(a.spec).read())
|
||
|
|
arms = [Arm(tag=x["tag"],
|
||
|
|
command=x["command"] if isinstance(x["command"], list)
|
||
|
|
else shlex.split(x["command"]),
|
||
|
|
env={k: str(v) for k, v in (x.get("env") or {}).items()})
|
||
|
|
for x in spec["arms"]]
|
||
|
|
keys = [k.strip() for k in a.keys.split(",") if k.strip()]
|
||
|
|
|
||
|
|
cores = parse_cores(a.cores) if a.cores else sorted(os.sched_getaffinity(0))
|
||
|
|
# Our own run puts ~len(cores) on the load average, so a threshold below
|
||
|
|
# that labels every pass contended and tells us nothing.
|
||
|
|
busy_load = a.busy_load if a.busy_load is not None else len(cores) + 8.0
|
||
|
|
check = preflight(cores)
|
||
|
|
if not check["clear"]:
|
||
|
|
detail = []
|
||
|
|
if check["stolen_pct"] > check["max_stolen_pct"]:
|
||
|
|
detail.append(f"{check['stolen_pct']}% of the cores' capacity is "
|
||
|
|
f"someone else's (limit {check['max_stolen_pct']}%)")
|
||
|
|
if check["busy_cores"]:
|
||
|
|
detail.append("sustained stragglers: "
|
||
|
|
+ ", ".join(f"{c}={p}%"
|
||
|
|
for c, p in check["busy_cores"].items()))
|
||
|
|
msg = "contended target cores: " + "; ".join(detail)
|
||
|
|
if not a.allow_busy:
|
||
|
|
raise SystemExit(
|
||
|
|
msg + "\n Another job is on these cores; this run would "
|
||
|
|
"measure the contention, not the code. Wait, pick other "
|
||
|
|
"cores with --cores, or pass --allow-busy to label the "
|
||
|
|
"result instead of trusting it.")
|
||
|
|
print(f"WARNING: {msg} (--allow-busy)")
|
||
|
|
|
||
|
|
print(f"{len(arms)} arms x {a.passes} passes, rotating; metric={a.metric}; "
|
||
|
|
f"{len(cores)} cores, pre-flight "
|
||
|
|
f"{'clear' if check['clear'] else 'BUSY'} "
|
||
|
|
f"({check['stolen_pct']}% stolen), "
|
||
|
|
f"busy-label above load {busy_load:.0f}")
|
||
|
|
|
||
|
|
lock = contextlib.nullcontext() if a.no_lock else HostLock(note=a.metric)
|
||
|
|
with lock:
|
||
|
|
observations = run_grid(arms, passes=a.passes, metric=a.metric,
|
||
|
|
timeout=a.timeout, keys=keys,
|
||
|
|
settle_s=a.settle_s, busy_load=busy_load)
|
||
|
|
summaries = summarize(observations, a.metric, a.noise_threshold)
|
||
|
|
print("\n" + format_summary(summaries))
|
||
|
|
|
||
|
|
checks = []
|
||
|
|
if a.check:
|
||
|
|
checks = check_expectations(summaries, json.loads(open(a.check).read()))
|
||
|
|
print("\nregression check\n" + format_check(checks))
|
||
|
|
|
||
|
|
if a.out:
|
||
|
|
with open(a.out, "w") as f:
|
||
|
|
json.dump({
|
||
|
|
"metric": a.metric, "passes": a.passes,
|
||
|
|
"lower_is_better": metric_is_lower_better(a.metric),
|
||
|
|
"provenance": provenance({
|
||
|
|
"arms": [{"tag": x.tag, "command": x.command, "env": x.env}
|
||
|
|
for x in arms],
|
||
|
|
"preflight": check,
|
||
|
|
"busy_load_threshold": busy_load,
|
||
|
|
}),
|
||
|
|
"observations": [vars(o) for o in observations],
|
||
|
|
"summary": [vars(s) for s in summaries],
|
||
|
|
"checks": checks,
|
||
|
|
}, f, indent=2)
|
||
|
|
print(f"\nwrote {a.out}")
|
||
|
|
|
||
|
|
regressions = [c for c in checks if c["status"] == "REGRESSION"]
|
||
|
|
if regressions:
|
||
|
|
raise SystemExit(
|
||
|
|
f"{len(regressions)} regression(s): "
|
||
|
|
+ ", ".join(f"{c['arm']}/{c['metric']} {c['drift_pct']:+.2f}%"
|
||
|
|
for c in regressions))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|