[CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5845)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
wangyuandamy-why-3459 authored and GitHub committed 2026-08-06 21:20:04 +08:00
1 parent 48298030d4
commit 4b625d1175
15 files changed
+414 -106

No files matched your search

@@ -21,10 +21,12 @@
"num-input-images": 0,
"enable-negative-prompt": false,
"baseline": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
"H100": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
}
}
},
"result": {
@@ -46,9 +46,11 @@
"median_audio_duration_s": 0,
"p99_audio_duration_s": 0,
"baseline": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
"H100": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
}
},
"random_input_len": 0,
"random_output_len": 0
+4 -2
View File
@@ -237,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
break
# QPS / request-rate sweep
for qps, num_prompt in zip(qps_list, num_prompt_list):
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
args=args,
@@ -246,6 +246,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
@@ -253,7 +254,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
assert_result(result, num_prompt)
# concurrency test
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
args=args,
@@ -262,6 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
@@ -23,7 +23,6 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u
the same per-file aggregation; ``-m`` only selects which cases run.
"""
import copy
import json
import os
import socket
@@ -45,6 +44,7 @@ from tests.dfx.conftest import (
get_runtime_resource_label,
hardware_json_value,
is_diffusion_perf_config,
resolve_baseline_for_sweep,
resolve_pytest_marks,
resource_label_for_filename,
)
@@ -766,8 +766,8 @@ def run_benchmark(
completed = metrics.get("completed_requests", metrics.get("completed", 0))
failed = metrics.get("failed_requests", metrics.get("failed", 0))
# Persist the JSON baseline as-is (no hardware / sweep resolution).
baseline = copy.deepcopy(params.get("baseline") or {})
# Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
baseline = params.get("baseline") or {}
metrics["baseline"] = baseline
record: dict[str, Any] = {
@@ -862,6 +862,7 @@ def _build_run_params(
params: dict[str, Any],
*,
num_prompts: int,
sweep_index: int | None = None,
request_rate: Any | None = None,
max_concurrency: Any | None = None,
) -> dict[str, Any]:
@@ -873,6 +874,12 @@ def _build_run_params(
run_params["request-rate"] = request_rate
if max_concurrency is not None:
run_params["max-concurrency"] = max_concurrency
if "baseline" in params:
# Keep all hardware buckets; pick the metric value for this sweep step only.
run_params["baseline"] = resolve_baseline_for_sweep(
params.get("baseline"),
sweep_index=sweep_index,
)
return run_params
@@ -895,19 +902,20 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
sweep_runs: list[dict[str, Any]] = []
for request_rate, num_prompts in zip(request_rate_list, num_prompt_list):
for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
params,
request_rate=request_rate,
num_prompts=num_prompts,
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
}
)
for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list):
for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
@@ -915,6 +923,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
max_concurrency=max_concurrency,
num_prompts=num_prompts,
request_rate="inf",
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
}
@@ -927,6 +936,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
"params": _build_run_params(
params,
num_prompts=default_num_prompts,
sweep_index=0,
),
"num_prompts": default_num_prompts,
}
@@ -25,8 +25,8 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
"throughput_qps": [0.0303, 0.0302, 0.0302],
"latency_mean": [32.6, 62.2, 111.7],
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
@@ -25,8 +25,8 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
"throughput_qps": [0.0549, 0.0547, 0.0549],
"latency_mean": [18.0, 34.3, 61.4],
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
@@ -134,7 +134,7 @@
"max-concurrency": 16,
"num-prompts": 96,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
@@ -38,7 +38,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.21,
"latency_p99": 4.7469,
"peak_memory_mb_max": 101100,
@@ -38,7 +38,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.20,
"latency_p99": 5.1025,
"peak_memory_mb_max": 97402,
@@ -37,7 +37,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.213,
"latency_mean": 4.6953,
"latency_p99": 4.8601,
+184 -35
View File
@@ -17,7 +17,7 @@ def test_task_excluded_from_cli_args():
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"percentile-metrics": "audio_rtf,audio_ttfp",
"baseline": {"mean_audio_rtf": [0.5]},
"baseline": {"H100": {"mean_audio_rtf": [0.5]}},
}
exclude_keys = {
"request_rate",
@@ -123,35 +123,6 @@ def test_resolve_pytest_marks_rejects_legacy_object_format():
)
def test_extract_mark_resource_label():
from tests.dfx.conftest import extract_mark_resource_label
assert extract_mark_resource_label(None) == "na"
assert (
extract_mark_resource_label(
[
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
"full_model",
]
)
== "H100"
)
assert (
extract_mark_resource_label(
[
{
"hardware_marks": {
"res": {"cuda": "H100", "rocm": "MI325"},
"num_cards": 2,
}
},
"diffusion",
]
)
== "H100-MI325"
)
def test_resource_label_for_filename():
from tests.dfx.conftest import resource_label_for_filename
@@ -169,16 +140,14 @@ def test_hardware_json_value():
assert hardware_json_value(None) == ""
def test_extract_configs_resource_label(monkeypatch):
from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
def test_get_runtime_resource_label(monkeypatch):
from tests.dfx.conftest import get_runtime_resource_label
monkeypatch.setattr(
"tests.dfx.conftest._read_runtime_device_name",
lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
)
get_runtime_resource_label(refresh=True)
assert extract_configs_resource_label([]) == "H100"
get_runtime_resource_label(refresh=True)
assert get_runtime_resource_label(refresh=True) == "H100"
monkeypatch.setattr(
"tests.dfx.conftest._read_runtime_device_name",
lambda *, device_id=0: "Ascend910B2",
@@ -336,3 +305,183 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
("start", tts_server),
("stop", tts_server),
]
def test_is_hardware_nested_baseline():
from tests.dfx.conftest import (
_RUNTIME_DEVICE_ALIASES,
is_hardware_nested_baseline,
)
from tests.helpers.mark import get_hardware_mark_list
hardware_marks = get_hardware_mark_list()
assert hardware_marks
assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks
assert "cuda" not in hardware_marks
assert "full_model" not in hardware_marks
# Runtime aliases are a full static list (independent of baseline allowlist).
assert "H100" in _RUNTIME_DEVICE_ALIASES
assert "A100" in _RUNTIME_DEVICE_ALIASES
assert is_hardware_nested_baseline(
{
"H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]},
"A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]},
}
)
# Custom metric names under a known hardware label are allowed.
assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}})
assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}})
# Flat metric maps / concurrency-keyed maps are not hardware-nested.
assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]})
assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}})
assert not is_hardware_nested_baseline({})
# Alias-only labels (not [hardware-resource] markers) are rejected for baselines.
assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}})
# Unknown top-level hardware label is rejected.
assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}})
# Empty per-hardware metric map is rejected.
assert not is_hardware_nested_baseline({"H100": {}})
def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency():
from tests.dfx.conftest import resolve_baseline_for_sweep
baseline = {
"H100": {
"mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8],
"mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0],
},
"A3": {
"mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0],
"mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0],
},
}
# max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32
got = resolve_baseline_for_sweep(baseline, sweep_index=4)
assert got == {
"H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0},
"A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0},
}
# First sweep step keeps both hardware buckets too.
got0 = resolve_baseline_for_sweep(baseline, sweep_index=0)
assert set(got0) == {"H100", "A3"}
assert got0["H100"]["mean_ttft_ms"] == 96.4
assert got0["A3"]["mean_ttft_ms"] == 90.0
def test_resolve_baseline_for_sweep_rejects_flat_baseline():
from tests.dfx.conftest import resolve_baseline_for_sweep
with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"):
resolve_baseline_for_sweep(
{"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]},
sweep_index=1,
)
def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label():
from tests.dfx.conftest import resolve_baseline_for_sweep
with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"):
resolve_baseline_for_sweep(
{"A100": {"throughput_qps": 1.0}},
sweep_index=0,
)
def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware():
from tests.dfx.conftest import resolve_baseline_for_sweep
# Sweep-aligned lists under each hardware bucket (canonical form).
listed = {
"H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]},
"A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]},
}
assert resolve_baseline_for_sweep(listed, sweep_index=1) == {
"H100": {"throughput_qps": 0.6, "latency_mean": 2.0},
"A3": {"throughput_qps": 0.7, "latency_mean": 1.8},
}
# Scalars under hardware stay as-is (single-concurrency cases).
scalar = {"H100": {"throughput_qps": 0.5}}
assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}}
# Custom metric names are preserved.
custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}}
assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}}
assert resolve_baseline_for_sweep(None) == {}
assert resolve_baseline_for_sweep({}) == {}
def test_resolve_baseline_value_errors():
from tests.dfx.conftest import resolve_baseline_value
with pytest.raises(ValueError, match="sweep_index"):
resolve_baseline_value([1.0, 2.0], sweep_index=None)
with pytest.raises(IndexError):
resolve_baseline_value([1.0], sweep_index=1)
with pytest.raises(TypeError, match="not supported"):
resolve_baseline_value({"1": 0.4}, sweep_index=0)
def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch):
"""``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency."""
import importlib
import sys
cfg = tmp_path / "mini_diffusion_perf.json"
cfg.write_text(
json.dumps(
[
{
"test_name": "test_mini",
"server_type": "vllm-omni",
"mark": [
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
"diffusion",
"full_model",
],
"server_params": {"model": "m/mini"},
"benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}],
}
]
),
encoding="utf-8",
)
monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)])
sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None)
from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb
importlib.reload(rdb)
params = {
"name": "c_sweep",
"dataset": "random",
"task": "t2i",
"num-prompts": [8, 16, 32],
"max-concurrency": [1, 8, 32],
"baseline": {
"H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]},
"A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]},
},
}
run32 = rdb._build_run_params(
params,
num_prompts=32,
max_concurrency=32,
request_rate="inf",
sweep_index=2,
)
assert run32["max-concurrency"] == 32
assert run32["baseline"] == {
"H100": {"throughput_qps": 0.3, "latency_mean": 30.0},
"A3": {"throughput_qps": 0.33, "latency_mean": 29.0},
}
sweeps = rdb._iter_sweep_runs(params)
assert len(sweeps) == 3
assert sweeps[2]["params"]["max-concurrency"] == 32
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
assert "A3" in sweeps[2]["params"]["baseline"]
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)