[Perf] Enhance benchmark script to support baseline thresholds and proved result handling (#2789)

This commit is contained in:
wangyu authored and GitHub committed 2026-04-15 10:28:28 +08:00
1 parent 02e5dc747d
commit a782ae4780
4 files changed
+162 -32

No files matched your search

+36 -13
View File
@@ -319,10 +319,10 @@ def _load_json_file(path: str) -> dict[str, Any] | list[Any] | None:
def _parse_from_filename(filename: str) -> dict[str, Any]:
"""Parse test-related metadata from a result JSON filename.
"""Parse test-related metadata from a ``result_test_*.json`` filename.
Expected pattern (after prefix/suffix stripped):
<test_name>_<dataset_name>_<max_concurrency>_<num_prompts>_<timestamp>
Matches ``tests/dfx/perf/scripts/run_benchmark.py`` naming, including optional
``_in{X}_out{Y}_`` before the timestamp (``na`` when unset).
"""
name, ext = os.path.splitext(filename)
if ext != ".json" or not name.startswith(_RESULT_JSON_PREFIX):
@@ -331,22 +331,42 @@ def _parse_from_filename(filename: str) -> dict[str, Any]:
core = name[len(_RESULT_JSON_PREFIX) :]
parts = core.split("_")
if len(parts) < 5:
LOGGER.warning("filename '%s' does not match expected pattern, skip parsing test metadata", filename)
LOGGER.warning(
"filename '%s' does not match expected pattern (need >= 5 segments), skip parsing",
filename,
)
return {}
timestamp = parts[-1]
num_prompts_str = parts[-2]
max_concurrency_str = parts[-3]
dataset_name = parts[-4]
test_name = "_".join(parts[:-4]) if parts[:-4] else ""
idx = len(parts) - 1
timestamp = parts[idx]
idx -= 1
parsed: dict[str, Any] = {}
if len(timestamp) >= 15:
parsed["date"] = timestamp
if dataset_name in DATASET_NAME_ALLOWED:
parsed["dataset_name"] = dataset_name
if idx >= 0 and parts[idx].startswith("out"):
parsed["random_output_len"] = parts[idx][3:]
idx -= 1
if idx >= 0 and parts[idx].startswith("in"):
parsed["random_input_len"] = parts[idx][2:]
idx -= 1
if idx < 3:
LOGGER.warning(
"filename '%s' has too few segments after timestamp / optional in-out (idx=%s)",
filename,
idx,
)
return parsed
num_prompts_str = parts[idx]
idx -= 1
flow_str = parts[idx]
idx -= 1
dataset_name = parts[idx]
idx -= 1
test_name = "_".join(parts[: idx + 1]) if idx >= 0 else ""
try:
parsed["num_prompts"] = int(num_prompts_str)
@@ -354,13 +374,16 @@ def _parse_from_filename(filename: str) -> dict[str, Any]:
pass
try:
parsed["max_concurrency"] = int(max_concurrency_str)
parsed["max_concurrency"] = int(flow_str)
except (TypeError, ValueError):
pass
if test_name:
parsed["test_name"] = test_name
if dataset_name in DATASET_NAME_ALLOWED:
parsed["dataset_name"] = dataset_name
return parsed
+36 -9
View File
@@ -67,6 +67,7 @@ def _load_json_file(path: str) -> dict[str, Any] | None:
def _parse_from_filename(filename: str) -> dict[str, Any]:
"""Parse ``result_test_*.json`` filenames; same rules as ``generate_nightly_perf_excel``."""
name, ext = os.path.splitext(filename)
if ext != ".json" or not name.startswith(_RESULT_JSON_PREFIX):
return {}
@@ -75,32 +76,58 @@ def _parse_from_filename(filename: str) -> dict[str, Any]:
parts = core.split("_")
if len(parts) < 5:
LOGGER.warning(
"filename '%s' does not match expected pattern, skip parsing test metadata",
"filename '%s' does not match expected pattern (need >= 5 segments), skip parsing",
filename,
)
return {}
timestamp = parts[-1]
num_prompts_str = parts[-2]
max_concurrency_str = parts[-3]
dataset_name = parts[-4]
test_name = "_".join(parts[:-4]) if parts[:-4] else ""
idx = len(parts) - 1
timestamp = parts[idx]
idx -= 1
parsed: dict[str, Any] = {}
if len(timestamp) >= 15:
parsed["date"] = timestamp
if dataset_name in ("random", "random-mm"):
parsed["dataset_name"] = dataset_name
if idx >= 0 and parts[idx].startswith("out"):
parsed["random_output_len"] = parts[idx][3:]
idx -= 1
if idx >= 0 and parts[idx].startswith("in"):
parsed["random_input_len"] = parts[idx][2:]
idx -= 1
if idx < 3:
LOGGER.warning(
"filename '%s' has too few segments after timestamp / optional in-out (idx=%s)",
filename,
idx,
)
return parsed
num_prompts_str = parts[idx]
idx -= 1
flow_str = parts[idx]
idx -= 1
dataset_name = parts[idx]
idx -= 1
test_name = "_".join(parts[: idx + 1]) if idx >= 0 else ""
try:
parsed["num_prompts"] = int(num_prompts_str)
except (TypeError, ValueError):
pass
try:
parsed["max_concurrency"] = int(max_concurrency_str)
parsed["max_concurrency"] = int(flow_str)
except (TypeError, ValueError):
pass
if test_name:
parsed["test_name"] = test_name
if dataset_name in ("random", "random-mm"):
parsed["dataset_name"] = dataset_name
return parsed