182 lines
5.8 KiB
Python
182 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|||
|
|
"""Render the committed edge results (stream_latency / init_profile / hw_emulation / cpu_sweep /
|
||
|
|
codec_stream) as markdown tables.
|
||
|
|
|
||
|
|
python benchmarks/tts/edge_results/summarize.py [--root benchmarks/tts/edge_results] [--out summary.md]
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
HERE = Path(__file__).resolve().parent
|
||
|
|
GiB = 1 << 30
|
||
|
|
|
||
|
|
|
||
|
|
def _load(p: Path) -> dict[str, Any] | None:
|
||
|
|
try:
|
||
|
|
return json.loads(p.read_text())
|
||
|
|
except (OSError, ValueError):
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _f(v: Any, nd: int = 1) -> str:
|
||
|
|
if v is None:
|
||
|
|
return ""
|
||
|
|
if isinstance(v, float):
|
||
|
|
return f"{v:.{nd}f}"
|
||
|
|
return str(v)
|
||
|
|
|
||
|
|
|
||
|
|
def stream_latency_rows(root: Path) -> list[dict[str, Any]]:
|
||
|
|
rows = []
|
||
|
|
for f in sorted((root / "stream_latency").glob("*/*/*.json")):
|
||
|
|
d = _load(f)
|
||
|
|
if not d or "summary" not in d:
|
||
|
|
continue
|
||
|
|
s = d["summary"]
|
||
|
|
meta = d.get("meta", {})
|
||
|
|
res = d.get("resources", {}) or {}
|
||
|
|
tag = f.stem.split("_", 1)[1] if "_" in f.stem else ""
|
||
|
|
st = d.get("step_stats") or {}
|
||
|
|
rows.append(
|
||
|
|
{
|
||
|
|
"model": f.parents[1].name.replace("Qwen3-TTS-12Hz-", "").replace("-CustomVoice", ""),
|
||
|
|
"deploy": f.parent.name,
|
||
|
|
"tag": tag,
|
||
|
|
"run": f.stem.split("_", 1)[0],
|
||
|
|
"n": s.get("n_requests"),
|
||
|
|
"init_s": d.get("init", {}).get("init_s"),
|
||
|
|
"ttfa_p50": s["ttfa_ms"]["p50"],
|
||
|
|
"ttfa_p90": s["ttfa_ms"]["p90"],
|
||
|
|
"playback_p50": s["playback_start_ms"]["p50"],
|
||
|
|
"stall_p50": s["stall_at_ttfa_ms"]["p50"],
|
||
|
|
"rtf_stream_p50": s["rtf_streamed"]["p50"],
|
||
|
|
"rtf_total_p50": s["rtf_total"]["p50"],
|
||
|
|
"rss_gib": (res.get("peak_rss_bytes") or 0) / GiB,
|
||
|
|
"gpu_mib": res.get("peak_gpu_mem_mib"),
|
||
|
|
"sha": (meta.get("git_sha") or "")[:8],
|
||
|
|
"step_stats": st.get("attribution", [])[:6] if st else None,
|
||
|
|
"path": str(f.relative_to(root)),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def table(rows: list[dict[str, Any]], cols: list[str], nd: int = 1) -> str:
|
||
|
|
out = ["| " + " | ".join(cols) + " |", "|" + "---|" * len(cols)]
|
||
|
|
for r in rows:
|
||
|
|
out.append("| " + " | ".join(_f(r.get(c), nd) for c in cols) + " |")
|
||
|
|
return "\n".join(out)
|
||
|
|
|
||
|
|
|
||
|
|
def render(root: Path) -> str:
|
||
|
|
parts = ["# Edge results summary\n"]
|
||
|
|
rows = stream_latency_rows(root)
|
||
|
|
parts.append("## Streaming latency (stream_latency_bench.py)\n")
|
||
|
|
parts.append(
|
||
|
|
table(
|
||
|
|
rows,
|
||
|
|
[
|
||
|
|
"model",
|
||
|
|
"deploy",
|
||
|
|
"tag",
|
||
|
|
"run",
|
||
|
|
"n",
|
||
|
|
"init_s",
|
||
|
|
"ttfa_p50",
|
||
|
|
"ttfa_p90",
|
||
|
|
"playback_p50",
|
||
|
|
"stall_p50",
|
||
|
|
"rtf_stream_p50",
|
||
|
|
"rtf_total_p50",
|
||
|
|
"rss_gib",
|
||
|
|
"gpu_mib",
|
||
|
|
"sha",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
)
|
||
|
|
for r in rows:
|
||
|
|
if r.get("step_stats"):
|
||
|
|
parts.append(f"\n### Step attribution: {r['model']} {r['deploy']} {r['tag']} ({r['run']})\n")
|
||
|
|
parts.append(
|
||
|
|
table(r["step_stats"], ["role", "counter", "count", "mean_ms", "p95_ms", "share_of_frame"], nd=3)
|
||
|
|
)
|
||
|
|
for name, cols in (
|
||
|
|
("init_profile", ["config", "init_s", "ttfa_ms", "gpu_mib"]),
|
||
|
|
(
|
||
|
|
"hw_emulation",
|
||
|
|
[
|
||
|
|
"case",
|
||
|
|
"hardware_class",
|
||
|
|
"returncode",
|
||
|
|
"init_s",
|
||
|
|
"ttfa_p50",
|
||
|
|
"playback_start_p50",
|
||
|
|
"stall_p50",
|
||
|
|
"rtf_total_p50",
|
||
|
|
"peak_rss_gib",
|
||
|
|
"peak_gpu_mib",
|
||
|
|
],
|
||
|
|
),
|
||
|
|
(
|
||
|
|
"cpu_sweep",
|
||
|
|
[
|
||
|
|
"case",
|
||
|
|
"threads",
|
||
|
|
"inductor",
|
||
|
|
"returncode",
|
||
|
|
"init_s",
|
||
|
|
"ttfa_p50",
|
||
|
|
"playback_start_p50",
|
||
|
|
"rtf_total_p50",
|
||
|
|
"peak_rss_gib",
|
||
|
|
],
|
||
|
|
),
|
||
|
|
):
|
||
|
|
for f in sorted((root / name).glob("*/summary.json")):
|
||
|
|
d = _load(f)
|
||
|
|
if not d:
|
||
|
|
continue
|
||
|
|
parts.append(f"\n## {name} {f.parent.name}\n")
|
||
|
|
rows_n = d.get("rows") or d.get("configs") or []
|
||
|
|
if name == "init_profile" and isinstance(rows_n, dict):
|
||
|
|
rows_n = [{"config": k, **v} for k, v in rows_n.items()]
|
||
|
|
parts.append(table(rows_n, cols))
|
||
|
|
if name == "cpu_sweep":
|
||
|
|
for r in rows_n:
|
||
|
|
if r.get("step_stats"):
|
||
|
|
parts.append(f"\n### Step attribution: {r['case']}\n")
|
||
|
|
parts.append(
|
||
|
|
table(
|
||
|
|
r["step_stats"],
|
||
|
|
["role", "counter", "count", "mean_ms", "p95_ms", "share_of_frame"],
|
||
|
|
nd=3,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
for f in sorted((root / "codec_stream").glob("**/*.json")):
|
||
|
|
d = _load(f)
|
||
|
|
if d and ("snr_db" in d or "parity" in d):
|
||
|
|
parts.append(f"\n## codec_stream {f.relative_to(root)}\n")
|
||
|
|
parts.append("```json\n" + json.dumps(d, indent=1)[:3000] + "\n```")
|
||
|
|
return "\n".join(parts) + "\n"
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv: list[str] | None = None) -> int:
|
||
|
|
p = argparse.ArgumentParser(description=__doc__)
|
||
|
|
p.add_argument("--root", default=str(HERE))
|
||
|
|
p.add_argument("--out", default=None)
|
||
|
|
args = p.parse_args(argv)
|
||
|
|
text = render(Path(args.root))
|
||
|
|
if args.out:
|
||
|
|
Path(args.out).write_text(text)
|
||
|
|
else:
|
||
|
|
print(text)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|