anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI] Nightly Benchmark - Add an HTML generator, Update the EXCEL generator. (#1831)

Browse Source 
Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
Alicia authored and GitHub committed 2026-03-18 20:13:57 +08:00
1 parent 61e170c69a
commit 4a44f6ee5c
2 files changed
+2161 -42
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/nightly/generate_nightly_perf_excel.py
+353 -42
View File
@@ -22,6 +22,49 @@ LOGGER = logging.getLogger(__name__)
GREY_BLOCK_FILL = PatternFill(start_color="D3D3D3", fill_type="solid")
# Diffusion sheet columns (Qwen-Image diffusion benchmark).
DIFFUSION_BENCHMARK_COLUMNS: tuple[str, ...] = (
"duration",
"completed_requests",
"failed_requests",
"throughput_qps",
"latency_mean",
"latency_median",
"latency_p50",
"latency_p99",
"peak_memory_mb_max",
"peak_memory_mb_mean",
"peak_memory_mb_median",
"slo_attainment_rate",
)
DIFFUSION_NUMERIC_FORMAT_COLUMNS: tuple[str, ...] = DIFFUSION_BENCHMARK_COLUMNS
DIFFUSION_SUMMARY_COLUMNS: tuple[str, ...] = (
"date",
"test_name",
"model",
"backend",
"dataset",
"task",
"completed_requests",
"failed_requests",
"duration",
"throughput_qps",
"latency_mean",
"latency_median",
"latency_p50",
"latency_p99",
"peak_memory_mb_max",
"peak_memory_mb_mean",
"peak_memory_mb_median",
"slo_attainment_rate",
"commit_sha",
"build_id",
"build_url",
"source_file",
)
# Benchmark metric columns: grey the latest row's cell when value changed vs previous date.
BENCHMARK_COLUMNS: tuple[str, ...] = (
"num_prompts",
@@ -53,10 +96,29 @@ BENCHMARK_COLUMNS: tuple[str, ...] = (
NUMERIC_FORMAT_COLUMNS: tuple[str, ...] = tuple(
c for c in BENCHMARK_COLUMNS if c not in ("request_rate", "max_concurrency")
)
DATASET_NAME_ALLOWED = ("random", "random-mm")
_COLUMNS_FILENAME = "nightly_perf_summary_columns.txt"
_RESULT_JSON_PREFIX = "result_test_"
_DIFFUSION_JSON_PREFIX = "diffusion_perf_"
DEFAULT_INPUT_DIR = os.getenv("DEFAULT_INPUT_DIR") if os.getenv("DEFAULT_INPUT_DIR") else "tests"
DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") if os.getenv("DEFAULT_OUTPUT_DIR") else "tests"
DEFAULT_DIFFUSION_INPUT_DIR = os.getenv("DIFFUSION_BENCHMARK_DIR")
# Read omni/diffusion benchmarks from DEFAULT_INPUT_DIR, if DEFAULT_DIFFUSION_INPUT_DIR is not set.
def _omni_group_key(record: dict[str, Any]) -> tuple[Any, ...]:
return (
record.get("model_id") or "",
record.get("test_name") or "",
record.get("dataset_name") or "",
record.get("max_concurrency") if record.get("max_concurrency") is not None else 0,
record.get("num_prompts") if record.get("num_prompts") is not None else 0,
)
def _diffusion_group_key(record: dict[str, Any]) -> tuple[Any, ...]:
return (record.get("test_name") or "",)
def _load_summary_columns(script_dir: str) -> list[str]:
@@ -68,6 +130,8 @@ def _load_summary_columns(script_dir: str) -> list[str]:
"backend",
"model_id",
"tokenizer_id",
"test_name",
"dataset_name",
"num_prompts",
"request_rate",
"burstiness",
@@ -106,6 +170,33 @@ def _load_summary_columns(script_dir: str) -> list[str]:
return columns if columns else default
def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]:
"""Ensure omni summary contains required columns, even when a custom columns file exists."""
required = ("test_name", "dataset_name", "source_file")
existing = set(summary_columns)
if all(c in existing for c in required):
return summary_columns
out = list(summary_columns)
insert_after = "tokenizer_id"
if "test_name" not in existing:
if insert_after in existing:
idx = out.index(insert_after) + 1
out.insert(idx, "test_name")
else:
out.append("test_name")
existing = set(out)
if "dataset_name" not in existing:
if "test_name" in out:
idx = out.index("test_name") + 1
out.insert(idx, "dataset_name")
else:
out.append("dataset_name")
if "source_file" not in set(out):
out.append("source_file")
return out
def _vllm_omni_root() -> str:
"""Resolve vllm-omni repo root: directory that contains a 'tests' subdir (and usually 'tools')."""
path = os.path.dirname(os.path.abspath(__file__))
@@ -122,6 +213,11 @@ def _default_input_dir() -> str:
return os.path.join(root, DEFAULT_INPUT_DIR)
def _default_diffusion_input_dir(input_dir: str) -> str:
"""Default diffusion input dir: DIFFUSION_BENCHMARK_DIR if set, else fall back to omni input dir."""
return DEFAULT_DIFFUSION_INPUT_DIR if DEFAULT_DIFFUSION_INPUT_DIR else input_dir
def _default_output_file() -> str:
"""Default: vllm-omni root / DEFAULT_OUTPUT_DIR / nightly_perf_<timestamp>.xlsx."""
ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
@@ -139,12 +235,23 @@ def parse_args() -> argparse.Namespace:
default=_default_input_dir(),
help="Directory containing performance JSON files; default is <vllm-omni-root>/DEFAULT_INPUT_DIR.",
)
parser.add_argument(
"--diffusion-input-dir",
type=str,
default=None,
help=(
"Directory containing diffusion_perf_*.json files; default is "
"DIFFUSION_BENCHMARK_DIR, fallback to --input-dir."
),
)
parser.add_argument(
"--output-file",
type=str,
default=_default_output_file(),
help="Output path of the Excel report; \
default is <vllm-omni-root>/DEFAULT_OUTPUT_DIR/nightly_perf_<timestamp>.xlsx.",
help=(
"Output path of the Excel report; default is "
"<vllm-omni-root>/DEFAULT_OUTPUT_DIR/nightly_perf_<timestamp>.xlsx."
),
)
parser.add_argument(
"--commit-sha",
@@ -183,11 +290,54 @@ def _load_json_file(path: str) -> dict[str, Any] | None:
return data
def _iter_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
"""Iterate over JSON files in the input directory and yield normalized records.
commit_sha/build_id/build_url are not set here; they are applied later only to
rows with the latest date (see _apply_build_metadata_to_latest_only).
def _parse_from_filename(filename: str) -> dict[str, Any]:
"""Parse test-related metadata from a result JSON filename.
Expected pattern (after prefix/suffix stripped):
<test_name>_<dataset_name>_<max_concurrency>_<num_prompts>_<timestamp>
"""
name, ext = os.path.splitext(filename)
if ext != ".json" or not name.startswith(_RESULT_JSON_PREFIX):
return {}
core = name[len(_RESULT_JSON_PREFIX) :]
parts = core.split("_")
if len(parts) < 5:
LOGGER.warning("filename '%s' does not match expected pattern, skip parsing test metadata", filename)
return {}
timestamp = parts[-1]
num_prompts_str = parts[-2]
max_concurrency_str = parts[-3]
dataset_name = parts[-4]
test_name = "_".join(parts[:-4]) if parts[:-4] else ""
parsed: dict[str, Any] = {}
if len(timestamp) >= 15:
parsed["date"] = timestamp
if dataset_name in DATASET_NAME_ALLOWED:
parsed["dataset_name"] = dataset_name
try:
parsed["num_prompts"] = int(num_prompts_str)
except (TypeError, ValueError):
pass
try:
parsed["max_concurrency"] = int(max_concurrency_str)
except (TypeError, ValueError):
pass
if test_name:
parsed["test_name"] = test_name
return parsed
def _iter_omni_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
"""Iterate over result_test_*.json files and yield normalized omni records."""
if not os.path.isdir(input_dir):
LOGGER.warning("input dir '%s' does not exist or is not a directory", input_dir)
return
@@ -195,6 +345,8 @@ def _iter_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
for entry in sorted(os.listdir(input_dir)):
if not entry.endswith(".json"):
continue
if not entry.startswith(_RESULT_JSON_PREFIX):
continue
full_path = os.path.join(input_dir, entry)
if not os.path.isfile(full_path):
continue
@@ -204,17 +356,89 @@ def _iter_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
continue
record: dict[str, Any] = dict(data)
record.setdefault("date", datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S"))
filename_meta = _parse_from_filename(os.path.basename(full_path))
if "date" not in record or not record["date"]:
if "date" in filename_meta:
record["date"] = filename_meta["date"]
else:
record["date"] = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
if "num_prompts" not in record or record["num_prompts"] is None:
if "num_prompts" in filename_meta:
record["num_prompts"] = filename_meta["num_prompts"]
if "max_concurrency" not in record or record["max_concurrency"] is None:
if "max_concurrency" in filename_meta:
record["max_concurrency"] = filename_meta["max_concurrency"]
if "test_name" not in record or not record.get("test_name"):
if "test_name" in filename_meta:
record["test_name"] = filename_meta["test_name"]
if "dataset_name" not in record or not record.get("dataset_name"):
if "dataset_name" in filename_meta:
record["dataset_name"] = filename_meta["dataset_name"]
record["source_file"] = os.path.basename(full_path)
yield record
def _parse_diffusion_from_filename(filename: str) -> dict[str, Any]:
"""Parse diffusion test_name/date from filename: diffusion_perf_<test_name>_<YYYYMMDD-HHMMSS>.json"""
name, ext = os.path.splitext(filename)
if ext != ".json" or not name.startswith(_DIFFUSION_JSON_PREFIX):
return {}
core = name[len(_DIFFUSION_JSON_PREFIX) :]
parts = core.split("_")
if len(parts) < 2:
return {}
timestamp = parts[-1]
test_name = "_".join(parts[:-1]) if parts[:-1] else ""
parsed: dict[str, Any] = {}
if len(timestamp) >= 15:
parsed["date"] = timestamp
if test_name:
parsed["test_name"] = test_name
return parsed
def _iter_diffusion_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
"""Iterate over diffusion_perf_*.json files and yield normalized diffusion records."""
if not os.path.isdir(input_dir):
LOGGER.warning("diffusion input dir '%s' does not exist or is not a directory", input_dir)
return
for entry in sorted(os.listdir(input_dir)):
if not entry.endswith(".json"):
continue
if not entry.startswith(_DIFFUSION_JSON_PREFIX):
continue
full_path = os.path.join(input_dir, entry)
if not os.path.isfile(full_path):
continue
data = _load_json_file(full_path)
if data is None:
continue
record: dict[str, Any] = dict(data)
filename_meta = _parse_diffusion_from_filename(os.path.basename(full_path))
if "date" not in record or not record.get("date"):
record["date"] = filename_meta.get("date") or datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
if "test_name" not in record or not record.get("test_name"):
if "test_name" in filename_meta:
record["test_name"] = filename_meta["test_name"]
record["source_file"] = os.path.basename(full_path)
yield record
def _collect_records(input_dir: str) -> list[dict[str, Any]]:
"""Collect all JSON records into a list."""
records: list[dict[str, Any]] = []
for record in _iter_json_records(input_dir):
records.append(record)
return records
return list(_iter_omni_json_records(input_dir))
def _collect_diffusion_records(diffusion_input_dir: str) -> list[dict[str, Any]]:
return list(_iter_diffusion_json_records(diffusion_input_dir))
def _apply_build_metadata_to_latest_only(
@@ -241,9 +465,17 @@ def _apply_build_metadata_to_latest_only(
def _sort_records_for_summary(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Sort so that same model_id is grouped, newest date first within each group."""
"""Sort so that same test configuration is grouped, newest date first within each group."""
by_date_desc = sorted(records, key=lambda r: (r.get("date") or ""), reverse=True)
return sorted(by_date_desc, key=lambda r: (r.get("model_id") or ""))
return sorted(
by_date_desc,
key=_omni_group_key,
)
def _sort_diffusion_records_for_summary(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
by_date_desc = sorted(records, key=lambda r: (r.get("date") or ""), reverse=True)
return sorted(by_date_desc, key=_diffusion_group_key)
def _values_differ(a: Any, b: Any) -> bool:
@@ -261,21 +493,22 @@ def _values_differ(a: Any, b: Any) -> bool:
return a != b
def _apply_benchmark_change_highlight(
def _apply_change_highlight(
ws,
summary_columns: Sequence[str],
columns: Sequence[str],
records: Sequence[dict[str, Any]],
benchmark_columns: Sequence[str],
group_key_fn,
) -> None:
"""Grey cells in the latest row of each model when a benchmark metric changed vs previous date."""
"""Grey cells in the latest row of each group when a metric changed vs previous row in the same group."""
if not records:
return
col_to_index = {c: i + 1 for i, c in enumerate(summary_columns)}
# Walk by model_id blocks (records already sorted by model_id, date desc).
col_to_index = {c: i + 1 for i, c in enumerate(columns)}
i = 0
while i < len(records):
model_id = records[i].get("model_id")
group_key = group_key_fn(records[i])
block_start = i
while i < len(records) and records[i].get("model_id") == model_id:
while i < len(records) and group_key_fn(records[i]) == group_key:
i += 1
block_end = i
newest_idx = block_start
@@ -283,7 +516,7 @@ def _apply_benchmark_change_highlight(
if prev_idx is None:
continue
excel_row = newest_idx + 2
for col in BENCHMARK_COLUMNS:
for col in benchmark_columns:
if col not in col_to_index:
continue
cur_val = records[newest_idx].get(col)
@@ -343,18 +576,20 @@ def _write_sheet(
ws.append(row_values)
def _format_benchmark_columns(
def _format_numeric_columns(
ws,
columns: Sequence[str],
numeric_columns: Sequence[str],
num_data_rows: int,
width: int,
) -> None:
"""Set number format and column width for numeric benchmark columns so values display (no ####)."""
numeric_set = set(NUMERIC_FORMAT_COLUMNS)
"""Set number format and column width for numeric columns so values display (no ####)."""
numeric_set = set(numeric_columns)
for c, col_name in enumerate(columns):
if col_name not in numeric_set:
continue
col_letter = get_column_letter(c + 1)
ws.column_dimensions[col_letter].width = 14
ws.column_dimensions[col_letter].width = width
for r in range(2, 2 + num_data_rows):
cell = ws.cell(row=r, column=c + 1)
if cell.value is not None and isinstance(cell.value, (int, float)):
@@ -367,6 +602,35 @@ def _format_benchmark_columns(
pass
def _set_column_width(ws, columns: Sequence[str], col_name: str, width: int) -> None:
"""Set a single column width by column name if present."""
try:
idx = list(columns).index(col_name)
except ValueError:
return
ws.column_dimensions[get_column_letter(idx + 1)].width = width
def _apply_column_widths(ws, columns: Sequence[str], widths: dict[str, int]) -> None:
for name, width in widths.items():
_set_column_width(ws, columns, name, width)
_OMNI_SUMMARY_WIDTHS = {
"endpoint_type": 10,
"backend": 10,
"model_id": 20,
"tokenizer_id": 20,
"test_name": 20,
"dataset_name": 10,
}
_DIFFUSION_SUMMARY_WIDTHS = {
"test_name": 30,
"model": 15,
}
def _ensure_parent_dir(path: str) -> None:
"""Ensure that the parent directory of the output file exists."""
parent = os.path.dirname(os.path.abspath(path))
@@ -377,34 +641,79 @@ def _ensure_parent_dir(path: str) -> None:
def generate_excel_report(
input_dir: str,
diffusion_input_dir: str,
output_file: str,
commit_sha: str | None,
build_id: str | None,
build_url: str | None,
) -> None:
"""Main logic: load JSON records and generate an Excel report."""
"""Main logic: load JSON records and generate an Excel report with multiple sheets."""
script_dir = os.path.dirname(os.path.abspath(__file__))
summary_columns = _load_summary_columns(script_dir)
omni_summary_columns = _ensure_omni_summary_columns(_load_summary_columns(script_dir))
records = _collect_records(input_dir)
if not records:
LOGGER.warning("no valid json records found under '%s'", input_dir)
omni_records = _collect_records(input_dir)
diffusion_records = _collect_diffusion_records(diffusion_input_dir)
sorted_records = _sort_records_for_summary(records)
_apply_build_metadata_to_latest_only(sorted_records, commit_sha, build_id, build_url)
if not omni_records:
LOGGER.warning("no valid omni json records found under '%s'", input_dir)
if not diffusion_records:
LOGGER.warning("no valid diffusion json records found under '%s'", diffusion_input_dir)
omni_sorted = _sort_records_for_summary(omni_records)
diffusion_sorted = _sort_diffusion_records_for_summary(diffusion_records)
_apply_build_metadata_to_latest_only(omni_sorted, commit_sha, build_id, build_url)
_apply_build_metadata_to_latest_only(diffusion_sorted, commit_sha, build_id, build_url)
wb = Workbook()
ws_summary = wb.active
ws_summary.title = "summary"
ws_omni_summary = wb.active
ws_omni_summary.title = "omni_summary"
_write_sheet(ws_omni_summary, omni_summary_columns, omni_sorted, numeric_columns=NUMERIC_FORMAT_COLUMNS)
_format_numeric_columns(
ws_omni_summary,
omni_summary_columns,
NUMERIC_FORMAT_COLUMNS,
len(omni_sorted),
width=14,
)
_apply_column_widths(ws_omni_summary, omni_summary_columns, _OMNI_SUMMARY_WIDTHS)
_apply_change_highlight(
ws_omni_summary,
omni_summary_columns,
omni_sorted,
BENCHMARK_COLUMNS,
_omni_group_key,
)
_write_sheet(ws_summary, summary_columns, sorted_records, numeric_columns=NUMERIC_FORMAT_COLUMNS)
_format_benchmark_columns(ws_summary, summary_columns, len(sorted_records))
_apply_benchmark_change_highlight(ws_summary, summary_columns, sorted_records)
ws_diff_summary = wb.create_sheet(title="diffusion_summary")
_write_sheet(
ws_diff_summary, DIFFUSION_SUMMARY_COLUMNS, diffusion_sorted, numeric_columns=DIFFUSION_NUMERIC_FORMAT_COLUMNS
)
_format_numeric_columns(
ws_diff_summary,
DIFFUSION_SUMMARY_COLUMNS,
DIFFUSION_NUMERIC_FORMAT_COLUMNS,
len(diffusion_sorted),
width=16,
)
_apply_column_widths(ws_diff_summary, DIFFUSION_SUMMARY_COLUMNS, _DIFFUSION_SUMMARY_WIDTHS)
_apply_change_highlight(
ws_diff_summary,
DIFFUSION_SUMMARY_COLUMNS,
diffusion_sorted,
DIFFUSION_BENCHMARK_COLUMNS,
_diffusion_group_key,
)
if sorted_records:
raw_columns = _build_raw_columns(sorted_records, summary_columns)
ws_raw = wb.create_sheet(title="raw")
_write_sheet(ws_raw, raw_columns, sorted_records)
if omni_sorted:
omni_raw_columns = _build_raw_columns(omni_sorted, omni_summary_columns)
ws_omni_raw = wb.create_sheet(title="omni_raw")
_write_sheet(ws_omni_raw, omni_raw_columns, omni_sorted)
if diffusion_sorted:
diffusion_raw_columns = _build_raw_columns(diffusion_sorted, DIFFUSION_SUMMARY_COLUMNS)
ws_diff_raw = wb.create_sheet(title="diffusion_raw")
_write_sheet(ws_diff_raw, diffusion_raw_columns, diffusion_sorted)
_ensure_parent_dir(output_file)
wb.save(output_file)
@@ -423,9 +732,11 @@ def main() -> None:
commit_sha = args.commit_sha or os.getenv("BUILDKITE_COMMIT")
build_id = args.build_id or os.getenv("BUILDKITE_BUILD_ID")
build_url = args.build_url or os.getenv("BUILDKITE_BUILD_URL")
diffusion_input_dir = args.diffusion_input_dir or _default_diffusion_input_dir(args.input_dir)
generate_excel_report(
input_dir=args.input_dir,
diffusion_input_dir=diffusion_input_dir,
output_file=args.output_file,
commit_sha=commit_sha,
build_id=build_id,
tools/nightly/generate_nightly_perf_html.py
+1808
View File
File diff suppressed because it is too large. Load diff