[CI][Perf] migrate diffusion DFX benches to vllm bench serve --omni (#7737)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
1 parent
a4ad73e08b
commit
0570b0da7e
30 files changed
+2040
-729
No files matched your search
@@ -1,4 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Generate a nightly Excel performance report from JSON results.
|
||||
|
||||
@@ -137,6 +140,7 @@ def _omni_group_key(record: dict[str, Any]) -> tuple[Any, ...]:
|
||||
return (
|
||||
record.get("model_id") or "",
|
||||
record.get("test_name") or "",
|
||||
record.get("name") or "",
|
||||
record.get("dataset_name") or "",
|
||||
record.get("max_concurrency") if record.get("max_concurrency") is not None else 0,
|
||||
record.get("num_prompts") if record.get("num_prompts") is not None else 0,
|
||||
@@ -171,6 +175,7 @@ def _load_summary_columns(script_dir: str) -> list[str]:
|
||||
"model_id",
|
||||
"tokenizer_id",
|
||||
"test_name",
|
||||
"name",
|
||||
"dataset_name",
|
||||
"num_prompts",
|
||||
"request_rate",
|
||||
@@ -218,7 +223,7 @@ def _load_summary_columns(script_dir: str) -> list[str]:
|
||||
|
||||
def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]:
|
||||
"""Ensure omni summary contains required columns, even when a custom columns file exists."""
|
||||
required = ("test_name", "dataset_name", "source_file")
|
||||
required = ("test_name", "name", "dataset_name", "source_file")
|
||||
existing = set(summary_columns)
|
||||
if all(c in existing for c in required):
|
||||
return summary_columns
|
||||
@@ -232,8 +237,18 @@ def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]:
|
||||
else:
|
||||
out.append("test_name")
|
||||
existing = set(out)
|
||||
if "dataset_name" not in existing:
|
||||
if "name" not in existing:
|
||||
if "test_name" in out:
|
||||
idx = out.index("test_name") + 1
|
||||
out.insert(idx, "name")
|
||||
else:
|
||||
out.append("name")
|
||||
existing = set(out)
|
||||
if "dataset_name" not in existing:
|
||||
if "name" in out:
|
||||
idx = out.index("name") + 1
|
||||
out.insert(idx, "dataset_name")
|
||||
elif "test_name" in out:
|
||||
idx = out.index("test_name") + 1
|
||||
out.insert(idx, "dataset_name")
|
||||
else:
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Generate an improved nightly HTML performance dashboard without modifying the
|
||||
existing generator.
|
||||
@@ -256,6 +259,7 @@ def _sort_omni_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
key=lambda r: (
|
||||
r.get("model_id") or "",
|
||||
r.get("test_name") or "",
|
||||
r.get("name") or "",
|
||||
r.get("dataset_name") or "",
|
||||
r.get("max_concurrency") or 0,
|
||||
r.get("num_prompts") or 0,
|
||||
@@ -1236,7 +1240,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
|
||||
const extraKey = prefix === "diff" ? "" : "tokenizer_id";
|
||||
const metaKeys = prefix === "diff"
|
||||
? ["test_name"]
|
||||
: ["test_name", "max_concurrency", "num_prompts"];
|
||||
: ["test_name", "name", "max_concurrency", "num_prompts"];
|
||||
const configFields = prefix === "diff"
|
||||
? ["test_name", "model", "endpoint", "dataset"]
|
||||
: [
|
||||
@@ -1245,13 +1249,14 @@ function initSection(prefix, columns, data, numericCols, groups) {{
|
||||
"model_id",
|
||||
"tokenizer_id",
|
||||
"test_name",
|
||||
"name",
|
||||
"dataset_name",
|
||||
"max_concurrency",
|
||||
"num_prompts",
|
||||
];
|
||||
const labelFields = prefix === "diff"
|
||||
? ["test_name", "dataset"]
|
||||
: ["test_name", "dataset_name"];
|
||||
: ["test_name", "name", "dataset_name"];
|
||||
const latestMetric = prefix === "diff" ? "throughput_qps" : "output_throughput";
|
||||
const secondaryMetric = prefix === "diff" ? "latency_mean" : "mean_e2el_ms";
|
||||
const modelList = document.getElementById(`${{prefix}}-model-list`);
|
||||
@@ -1650,7 +1655,7 @@ window.addEventListener("load", () => {{
|
||||
'<div class="section-subtitle">Chat/audio benchmark history with '
|
||||
"trend charts grouped by throughput, latency, and audio metrics."
|
||||
'</div></div><div class="section-pills"><div class="pill"><strong>'
|
||||
"Grouping</strong> test_name + dataset + concurrency + prompts</div>"
|
||||
"Grouping</strong> test_name + name + dataset + concurrency + prompts</div>"
|
||||
'<div class="pill"><strong>Snapshots</strong> single-point series '
|
||||
"stay readable</div></div></div>"
|
||||
),
|
||||
|
||||
@@ -41,7 +41,7 @@
|
||||
# From repo root: pytest -sv -m "<markers> and local_model" (markers from MODEL_TYPE: omni, tts,
|
||||
# diffusion; all → "(omni or tts or diffusion) and local_model"). Not filtered by nightly YAML.
|
||||
# LABEL_SUBSTR: if set, restrict to tests/**/test_*.py basenames and tests/dfx/perf/tests/*.json
|
||||
# whose filename contains the substring (benchmark runner chosen by JSON family).
|
||||
# whose filename contains the substring (benchmark runner via is_diffusion_perf_config).
|
||||
#
|
||||
# stability (when included in TEST_TYPE):
|
||||
# From repo root: pytest -s -v --run-level full_model -m "<mark>" tests/dfx/stability/scripts/...
|
||||
@@ -711,8 +711,19 @@ def perf_json_model_family(json_basename: str) -> str:
|
||||
|
||||
|
||||
def perf_json_runner(json_basename: str) -> Path:
|
||||
if perf_json_model_family(json_basename) in ("omni", "tts"):
|
||||
"""Pick benchmark runner from JSON schema (``dataset`` vs ``dataset_name``)."""
|
||||
json_path = REPO_ROOT / PERF_TESTS_REL / json_basename
|
||||
try:
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
from tests.dfx.conftest import is_diffusion_perf_config, load_configs
|
||||
|
||||
configs = load_configs(str(json_path))
|
||||
if configs and any(is_diffusion_perf_config(cfg) for cfg in configs):
|
||||
return RUN_DIFFUSION_BENCHMARK_REL
|
||||
return RUN_BENCHMARK_REL
|
||||
except Exception as exc:
|
||||
print(f"# warn: could not classify {json_basename}: {exc}", file=sys.stderr)
|
||||
return RUN_DIFFUSION_BENCHMARK_REL
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user