[CI][Perf] migrate diffusion DFX benches to vllm bench serve --omni (#7737)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-09-21 19:16:39 -07:00
1 parent a4ad73e08b
commit 0570b0da7e
30 files changed
+2040 -729

No files matched your search

+17 -2
View File
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Generate a nightly Excel performance report from JSON results.
@@ -137,6 +140,7 @@ def _omni_group_key(record: dict[str, Any]) -> tuple[Any, ...]:
return (
record.get("model_id") or "",
record.get("test_name") or "",
record.get("name") or "",
record.get("dataset_name") or "",
record.get("max_concurrency") if record.get("max_concurrency") is not None else 0,
record.get("num_prompts") if record.get("num_prompts") is not None else 0,
@@ -171,6 +175,7 @@ def _load_summary_columns(script_dir: str) -> list[str]:
"model_id",
"tokenizer_id",
"test_name",
"name",
"dataset_name",
"num_prompts",
"request_rate",
@@ -218,7 +223,7 @@ def _load_summary_columns(script_dir: str) -> list[str]:
def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]:
"""Ensure omni summary contains required columns, even when a custom columns file exists."""
required = ("test_name", "dataset_name", "source_file")
required = ("test_name", "name", "dataset_name", "source_file")
existing = set(summary_columns)
if all(c in existing for c in required):
return summary_columns
@@ -232,8 +237,18 @@ def _ensure_omni_summary_columns(summary_columns: list[str]) -> list[str]:
else:
out.append("test_name")
existing = set(out)
if "dataset_name" not in existing:
if "name" not in existing:
if "test_name" in out:
idx = out.index("test_name") + 1
out.insert(idx, "name")
else:
out.append("name")
existing = set(out)
if "dataset_name" not in existing:
if "name" in out:
idx = out.index("name") + 1
out.insert(idx, "dataset_name")
elif "test_name" in out:
idx = out.index("test_name") + 1
out.insert(idx, "dataset_name")
else:
+8 -3
View File
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Generate an improved nightly HTML performance dashboard without modifying the
existing generator.
@@ -256,6 +259,7 @@ def _sort_omni_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
key=lambda r: (
r.get("model_id") or "",
r.get("test_name") or "",
r.get("name") or "",
r.get("dataset_name") or "",
r.get("max_concurrency") or 0,
r.get("num_prompts") or 0,
@@ -1236,7 +1240,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
const extraKey = prefix === "diff" ? "" : "tokenizer_id";
const metaKeys = prefix === "diff"
? ["test_name"]
: ["test_name", "max_concurrency", "num_prompts"];
: ["test_name", "name", "max_concurrency", "num_prompts"];
const configFields = prefix === "diff"
? ["test_name", "model", "endpoint", "dataset"]
: [
@@ -1245,13 +1249,14 @@ function initSection(prefix, columns, data, numericCols, groups) {{
"model_id",
"tokenizer_id",
"test_name",
"name",
"dataset_name",
"max_concurrency",
"num_prompts",
];
const labelFields = prefix === "diff"
? ["test_name", "dataset"]
: ["test_name", "dataset_name"];
: ["test_name", "name", "dataset_name"];
const latestMetric = prefix === "diff" ? "throughput_qps" : "output_throughput";
const secondaryMetric = prefix === "diff" ? "latency_mean" : "mean_e2el_ms";
const modelList = document.getElementById(`${{prefix}}-model-list`);
@@ -1650,7 +1655,7 @@ window.addEventListener("load", () => {{
'<div class="section-subtitle">Chat/audio benchmark history with '
"trend charts grouped by throughput, latency, and audio metrics."
'</div></div><div class="section-pills"><div class="pill"><strong>'
"Grouping</strong> test_name + dataset + concurrency + prompts</div>"
"Grouping</strong> test_name + name + dataset + concurrency + prompts</div>"
'<div class="pill"><strong>Snapshots</strong> single-point series '
"stay readable</div></div></div>"
),
+13 -2
View File
@@ -41,7 +41,7 @@
# From repo root: pytest -sv -m "<markers> and local_model" (markers from MODEL_TYPE: omni, tts,
# diffusion; all → "(omni or tts or diffusion) and local_model"). Not filtered by nightly YAML.
# LABEL_SUBSTR: if set, restrict to tests/**/test_*.py basenames and tests/dfx/perf/tests/*.json
# whose filename contains the substring (benchmark runner chosen by JSON family).
# whose filename contains the substring (benchmark runner via is_diffusion_perf_config).
#
# stability (when included in TEST_TYPE):
# From repo root: pytest -s -v --run-level full_model -m "<mark>" tests/dfx/stability/scripts/...
@@ -711,8 +711,19 @@ def perf_json_model_family(json_basename: str) -> str:
def perf_json_runner(json_basename: str) -> Path:
if perf_json_model_family(json_basename) in ("omni", "tts"):
"""Pick benchmark runner from JSON schema (``dataset`` vs ``dataset_name``)."""
json_path = REPO_ROOT / PERF_TESTS_REL / json_basename
try:
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from tests.dfx.conftest import is_diffusion_perf_config, load_configs
configs = load_configs(str(json_path))
if configs and any(is_diffusion_perf_config(cfg) for cfg in configs):
return RUN_DIFFUSION_BENCHMARK_REL
return RUN_BENCHMARK_REL
except Exception as exc:
print(f"# warn: could not classify {json_basename}: {exc}", file=sys.stderr)
return RUN_DIFFUSION_BENCHMARK_REL