anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5845)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
wangyuandamy-why-3459 authored and GitHub committed 2026-08-06 21:20:04 +08:00
1 parent 48298030d4
commit 4b625d1175
15 files changed
+414 -106
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/contributing/ci/test_examples/l4_performance_tests.inc.md
+1 -1
View File
@@ -200,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par
| num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. |
| max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). |
| baseline | object | No | see below | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS.
| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. |
pyproject.toml
+10 -7
View File
@@ -250,13 +250,16 @@ markers = [
"npu: Tests that run on NPU/Ascend (auto-added)",
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
# specified computation resources marks (auto-added)
"H100: Tests that require H100 GPU",
"L4: Tests that require L4 GPU",
"MI325: Tests that require MI325 GPU (AMD/ROCm)",
"B60: Tests that require Intel Arc Pro B60 XPU",
"S5000: Tests that require S5000 GPU (Moore Threads/MUSA)",
"A2: Tests that require A2 NPU",
"A3: Tests that require A3 NPU",
# Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
"H100: [hardware-resource] Tests that require H100 GPU",
"H200: [hardware-resource] Tests that require H200 GPU",
"L4: [hardware-resource] Tests that require L4 GPU",
"B200: [hardware-resource] Tests that require B200 GPU",
"MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
"B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
"S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
"A2: [hardware-resource] Tests that require A2 NPU",
"A3: [hardware-resource] Tests that require A3 NPU",
"distributed_cuda: Tests that require multi cards on CUDA platform",
"distributed_rocm: Tests that require multi cards on ROCm platform",
"distributed_xpu: Tests that require multi cards on XPU platform",
tests/dfx/conftest.py
+121 -40
View File
@@ -1,4 +1,3 @@
import copy
import json
import os
import re
@@ -10,7 +9,7 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.mark import get_hardware_mark_list, hardware_marks
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.platforms import current_omni_platform
@@ -447,36 +446,16 @@ def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> li
return unique
def extract_mark_resource_label(mark_field: Any) -> str:
"""Return a filename-safe hardware label from ``mark.hardware_marks.res`` values.
Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``.
Prefer :func:`get_runtime_resource_label` for perf result filenames so labels
reflect the machine that actually ran the benchmark.
"""
if isinstance(mark_field, list):
for item in mark_field:
if isinstance(item, dict) and "hardware_marks" in item:
return extract_mark_resource_label(item)
return "na"
if not isinstance(mark_field, dict):
return "na"
hw = mark_field.get("hardware_marks")
if not isinstance(hw, dict):
return "na"
res = hw.get("res")
if not isinstance(res, dict) or not res:
return "na"
labels = [_safe_filename_token(value) for value in res.values()]
return "-".join(labels) if labels else "na"
_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = (
# Device marketing-name tokens for ``get_runtime_resource_label`` only.
# Longer / more specific tokens must appear before shorter substrings
# (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``).
_RUNTIME_DEVICE_ALIASES: tuple[str, ...] = (
"H100",
"H800",
"H200",
"H20",
"B200",
"GB200",
"L40S",
"L40",
"L4",
@@ -505,7 +484,7 @@ def _normalize_runtime_device_label(raw: str) -> str:
if not raw or not str(raw).strip():
return "na"
upper = str(raw).upper()
for token in _KNOWN_RUNTIME_RESOURCE_TOKENS:
for token in _RUNTIME_DEVICE_ALIASES:
if token.upper() in upper:
return _safe_filename_token(token)
compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw))
@@ -558,10 +537,108 @@ def resource_label_for_filename(resource_label: str | None) -> str:
return token
def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str:
"""Return runtime hardware label for perf result filenames."""
del configs
return get_runtime_resource_label()
def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool:
"""True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines.
Detection is structural + hardware-label allowlist (not metric-name hints):
- every top-level value is a non-empty dict (per-hardware metric map);
- every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list`
(``[hardware-resource]`` markers from ``pyproject.toml``).
Metric names under each hardware bucket are unconstrained.
Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated.
"""
if not baseline:
return False
if not all(isinstance(value, dict) and bool(value) for value in baseline.values()):
return False
allowed = {label.upper() for label in get_hardware_mark_list()}
return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys())
def resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
) -> Any:
"""Pick the baseline threshold for one metric at this sweep step.
Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar.
"""
if baseline_raw is None:
return None
if isinstance(baseline_raw, dict):
raise TypeError(
"per-metric baseline dict keyed by concurrency/request-rate is not supported; "
"use a sweep-aligned list or a scalar under each hardware bucket"
)
if isinstance(baseline_raw, (list, tuple)):
if sweep_index is None:
raise ValueError("list baseline requires sweep_index")
if not (0 <= sweep_index < len(baseline_raw)):
raise IndexError(f"baseline list len={len(baseline_raw)} has no index {sweep_index}")
return baseline_raw[sweep_index]
return baseline_raw
def resolve_baseline_for_sweep(
baseline_config: dict[str, Any] | None,
*,
sweep_index: int | None = None,
) -> dict[str, Any]:
"""Resolve sweep-dependent baselines while keeping every hardware bucket.
Expects hardware-nested input only::
{"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}}
with ``sweep_index=2`` (e.g. concurrency 32) becomes::
{"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}}
Per-metric values may be a sweep-aligned list or a scalar.
"""
if not baseline_config:
return {}
if not is_hardware_nested_baseline(baseline_config):
allowed = sorted(get_hardware_mark_list())
unknown = [
str(key).strip()
for key in baseline_config.keys()
if str(key).strip().upper() not in {label.upper() for label in allowed}
]
nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values())
if nested_shape and unknown:
raise ValueError(
"baseline top-level keys must be [hardware-resource] pytest markers from "
"pyproject.toml (tool.pytest.ini_options.markers). "
f"Unknown hardware label(s): {unknown!r}. "
f"Known markers: {allowed!r}. "
"Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' "
"to pyproject.toml, then retry."
)
raise ValueError(
"baseline must be hardware-nested with [hardware-resource] marker names as "
"top-level keys, e.g. "
'{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. '
f"got top-level keys={list(baseline_config.keys())!r}; "
f"known markers={allowed!r}. "
"If you need a new hardware bucket, add it to pyproject.toml markers with "
"the [hardware-resource] tag."
)
return {
hardware: {
metric_name: resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
)
for metric_name, baseline_raw in metrics.items()
}
for hardware, metrics in baseline_config.items()
if isinstance(metrics, dict)
}
def run_benchmark(
@@ -572,17 +649,18 @@ def run_benchmark(
num_prompt: int,
*,
baseline_config: dict[str, Any] | None = None,
sweep_index: int | None = None,
random_input_len: Any | None = None,
random_output_len: Any | None = None,
resource_label: str | None = None,
) -> dict[str, Any]:
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the raw
``baseline`` object from the test config (when ``baseline_config`` is provided),
including any hardware-nested map (e.g. ``{"H100": {...}}``). Downstream report
tooling can select a hardware bucket when needed. If the benchmark exits without
writing a result file, ``result_omni_template.json`` is used as a fallback.
After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the
sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested
maps keep every hardware bucket, but each metric is reduced to the value for
this concurrency / request-rate step. If the benchmark exits without writing a
result file, ``result_omni_template.json`` is used as a fallback.
"""
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
ri = _safe_filename_token(random_input_len)
@@ -649,8 +727,11 @@ def run_benchmark(
result = json.load(f)
if baseline_config:
# Persist the JSON baseline as-is (no hardware / sweep resolution).
result["baseline"] = copy.deepcopy(baseline_config)
# Keep every hardware bucket; resolve list metrics to this sweep step.
result["baseline"] = resolve_baseline_for_sweep(
baseline_config,
sweep_index=sweep_index,
)
else:
result["baseline"] = {}
if random_input_len is not None:
tests/dfx/perf/scripts/diffusion_result_template.json
+6 -4
View File
@@ -21,10 +21,12 @@
"num-input-images": 0,
"enable-negative-prompt": false,
"baseline": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
"H100": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
}
}
},
"result": {
tests/dfx/perf/scripts/result_omni_template.json
+5 -3
View File
@@ -46,9 +46,11 @@
"median_audio_duration_s": 0,
"p99_audio_duration_s": 0,
"baseline": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
"H100": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
}
},
"random_input_len": 0,
"random_output_len": 0
tests/dfx/perf/scripts/run_benchmark.py
+4 -2
View File
@@ -237,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
break
# QPS / request-rate sweep
for qps, num_prompt in zip(qps_list, num_prompt_list):
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
args=args,
@@ -246,6 +246,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
@@ -253,7 +254,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
assert_result(result, num_prompt)
# concurrency test
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
args=args,
@@ -262,6 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
tests/dfx/perf/scripts/run_diffusion_benchmark.py
+15 -5
View File
@@ -23,7 +23,6 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u
the same per-file aggregation; ``-m`` only selects which cases run.
"""
import copy
import json
import os
import socket
@@ -45,6 +44,7 @@ from tests.dfx.conftest import (
get_runtime_resource_label,
hardware_json_value,
is_diffusion_perf_config,
resolve_baseline_for_sweep,
resolve_pytest_marks,
resource_label_for_filename,
)
@@ -766,8 +766,8 @@ def run_benchmark(
completed = metrics.get("completed_requests", metrics.get("completed", 0))
failed = metrics.get("failed_requests", metrics.get("failed", 0))
# Persist the JSON baseline as-is (no hardware / sweep resolution).
baseline = copy.deepcopy(params.get("baseline") or {})
# Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
baseline = params.get("baseline") or {}
metrics["baseline"] = baseline
record: dict[str, Any] = {
@@ -862,6 +862,7 @@ def _build_run_params(
params: dict[str, Any],
*,
num_prompts: int,
sweep_index: int | None = None,
request_rate: Any | None = None,
max_concurrency: Any | None = None,
) -> dict[str, Any]:
@@ -873,6 +874,12 @@ def _build_run_params(
run_params["request-rate"] = request_rate
if max_concurrency is not None:
run_params["max-concurrency"] = max_concurrency
if "baseline" in params:
# Keep all hardware buckets; pick the metric value for this sweep step only.
run_params["baseline"] = resolve_baseline_for_sweep(
params.get("baseline"),
sweep_index=sweep_index,
)
return run_params
@@ -895,19 +902,20 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
sweep_runs: list[dict[str, Any]] = []
for request_rate, num_prompts in zip(request_rate_list, num_prompt_list):
for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
params,
request_rate=request_rate,
num_prompts=num_prompts,
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
}
)
for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list):
for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
@@ -915,6 +923,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
max_concurrency=max_concurrency,
num_prompts=num_prompts,
request_rate="inf",
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
}
@@ -927,6 +936,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
"params": _build_run_params(
params,
num_prompts=default_num_prompts,
sweep_index=0,
),
"num_prompts": default_num_prompts,
}
tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json
+2 -2
View File
@@ -25,8 +25,8 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
"throughput_qps": [0.0303, 0.0302, 0.0302],
"latency_mean": [32.6, 62.2, 111.7],
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
tests/dfx/perf/tests/test_boogu_image_vllm_omni.json
+2 -2
View File
@@ -25,8 +25,8 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
"throughput_qps": [0.0549, 0.0547, 0.0549],
"latency_mean": [18.0, 34.3, 61.4],
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
tests/dfx/perf/tests/test_hunyuan_image3_it2i.json
+1 -1
View File
@@ -134,7 +134,7 @@
"max-concurrency": 16,
"num-prompts": 96,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+1 -1
View File
@@ -38,7 +38,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.21,
"latency_p99": 4.7469,
"peak_memory_mb_max": 101100,
tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+1 -1
View File
@@ -38,7 +38,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.20,
"latency_p99": 5.1025,
"peak_memory_mb_max": 97402,
tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+1 -1
View File
@@ -37,7 +37,7 @@
"num-prompts": 10,
"max-concurrency": 1,
"baseline": {
"H100": {
"H200": {
"throughput_qps": 0.213,
"latency_mean": 4.6953,
"latency_p99": 4.8601,
tests/dfx/perf/tests/test_runner_metadata.py
+184 -35
View File
@@ -17,7 +17,7 @@ def test_task_excluded_from_cli_args():
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"percentile-metrics": "audio_rtf,audio_ttfp",
"baseline": {"mean_audio_rtf": [0.5]},
"baseline": {"H100": {"mean_audio_rtf": [0.5]}},
}
exclude_keys = {
"request_rate",
@@ -123,35 +123,6 @@ def test_resolve_pytest_marks_rejects_legacy_object_format():
)
def test_extract_mark_resource_label():
from tests.dfx.conftest import extract_mark_resource_label
assert extract_mark_resource_label(None) == "na"
assert (
extract_mark_resource_label(
[
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
"full_model",
]
)
== "H100"
)
assert (
extract_mark_resource_label(
[
{
"hardware_marks": {
"res": {"cuda": "H100", "rocm": "MI325"},
"num_cards": 2,
}
},
"diffusion",
]
)
== "H100-MI325"
)
def test_resource_label_for_filename():
from tests.dfx.conftest import resource_label_for_filename
@@ -169,16 +140,14 @@ def test_hardware_json_value():
assert hardware_json_value(None) == ""
def test_extract_configs_resource_label(monkeypatch):
from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
def test_get_runtime_resource_label(monkeypatch):
from tests.dfx.conftest import get_runtime_resource_label
monkeypatch.setattr(
"tests.dfx.conftest._read_runtime_device_name",
lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
)
get_runtime_resource_label(refresh=True)
assert extract_configs_resource_label([]) == "H100"
get_runtime_resource_label(refresh=True)
assert get_runtime_resource_label(refresh=True) == "H100"
monkeypatch.setattr(
"tests.dfx.conftest._read_runtime_device_name",
lambda *, device_id=0: "Ascend910B2",
@@ -336,3 +305,183 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
("start", tts_server),
("stop", tts_server),
]
def test_is_hardware_nested_baseline():
from tests.dfx.conftest import (
_RUNTIME_DEVICE_ALIASES,
is_hardware_nested_baseline,
)
from tests.helpers.mark import get_hardware_mark_list
hardware_marks = get_hardware_mark_list()
assert hardware_marks
assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks
assert "cuda" not in hardware_marks
assert "full_model" not in hardware_marks
# Runtime aliases are a full static list (independent of baseline allowlist).
assert "H100" in _RUNTIME_DEVICE_ALIASES
assert "A100" in _RUNTIME_DEVICE_ALIASES
assert is_hardware_nested_baseline(
{
"H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]},
"A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]},
}
)
# Custom metric names under a known hardware label are allowed.
assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}})
assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}})
# Flat metric maps / concurrency-keyed maps are not hardware-nested.
assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]})
assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}})
assert not is_hardware_nested_baseline({})
# Alias-only labels (not [hardware-resource] markers) are rejected for baselines.
assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}})
# Unknown top-level hardware label is rejected.
assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}})
# Empty per-hardware metric map is rejected.
assert not is_hardware_nested_baseline({"H100": {}})
def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency():
from tests.dfx.conftest import resolve_baseline_for_sweep
baseline = {
"H100": {
"mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8],
"mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0],
},
"A3": {
"mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0],
"mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0],
},
}
# max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32
got = resolve_baseline_for_sweep(baseline, sweep_index=4)
assert got == {
"H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0},
"A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0},
}
# First sweep step keeps both hardware buckets too.
got0 = resolve_baseline_for_sweep(baseline, sweep_index=0)
assert set(got0) == {"H100", "A3"}
assert got0["H100"]["mean_ttft_ms"] == 96.4
assert got0["A3"]["mean_ttft_ms"] == 90.0
def test_resolve_baseline_for_sweep_rejects_flat_baseline():
from tests.dfx.conftest import resolve_baseline_for_sweep
with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"):
resolve_baseline_for_sweep(
{"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]},
sweep_index=1,
)
def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label():
from tests.dfx.conftest import resolve_baseline_for_sweep
with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"):
resolve_baseline_for_sweep(
{"A100": {"throughput_qps": 1.0}},
sweep_index=0,
)
def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware():
from tests.dfx.conftest import resolve_baseline_for_sweep
# Sweep-aligned lists under each hardware bucket (canonical form).
listed = {
"H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]},
"A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]},
}
assert resolve_baseline_for_sweep(listed, sweep_index=1) == {
"H100": {"throughput_qps": 0.6, "latency_mean": 2.0},
"A3": {"throughput_qps": 0.7, "latency_mean": 1.8},
}
# Scalars under hardware stay as-is (single-concurrency cases).
scalar = {"H100": {"throughput_qps": 0.5}}
assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}}
# Custom metric names are preserved.
custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}}
assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}}
assert resolve_baseline_for_sweep(None) == {}
assert resolve_baseline_for_sweep({}) == {}
def test_resolve_baseline_value_errors():
from tests.dfx.conftest import resolve_baseline_value
with pytest.raises(ValueError, match="sweep_index"):
resolve_baseline_value([1.0, 2.0], sweep_index=None)
with pytest.raises(IndexError):
resolve_baseline_value([1.0], sweep_index=1)
with pytest.raises(TypeError, match="not supported"):
resolve_baseline_value({"1": 0.4}, sweep_index=0)
def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch):
"""``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency."""
import importlib
import sys
cfg = tmp_path / "mini_diffusion_perf.json"
cfg.write_text(
json.dumps(
[
{
"test_name": "test_mini",
"server_type": "vllm-omni",
"mark": [
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
"diffusion",
"full_model",
],
"server_params": {"model": "m/mini"},
"benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}],
}
]
),
encoding="utf-8",
)
monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)])
sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None)
from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb
importlib.reload(rdb)
params = {
"name": "c_sweep",
"dataset": "random",
"task": "t2i",
"num-prompts": [8, 16, 32],
"max-concurrency": [1, 8, 32],
"baseline": {
"H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]},
"A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]},
},
}
run32 = rdb._build_run_params(
params,
num_prompts=32,
max_concurrency=32,
request_rate="inf",
sweep_index=2,
)
assert run32["max-concurrency"] == 32
assert run32["baseline"] == {
"H100": {"throughput_qps": 0.3, "latency_mean": 30.0},
"A3": {"throughput_qps": 0.33, "latency_mean": 29.0},
}
sweeps = rdb._iter_sweep_runs(params)
assert len(sweeps) == 3
assert sweeps[2]["params"]["max-concurrency"] == 32
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
assert "A3" in sweeps[2]["params"]["baseline"]
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
tests/helpers/mark.py
+60 -1
View File
@@ -2,10 +2,69 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
from __future__ import annotations
import re
from functools import lru_cache
from pathlib import Path
import pytest
from vllm.platforms import current_platform
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
# Marker description tag in ``pyproject.toml`` ``tool.pytest.ini_options.markers``.
# Example: ``"H100: [hardware-resource] Tests that require H100 GPU"``.
_HARDWARE_RESOURCE_MARKER_TAG = "[hardware-resource]"
def _repo_root() -> Path:
return Path(__file__).resolve().parents[2]
def _load_pyproject() -> dict:
path = _repo_root() / "pyproject.toml"
try:
try:
import tomllib
except ModuleNotFoundError: # Python < 3.11
import tomli as tomllib # type: ignore[no-redef]
with path.open("rb") as f:
return tomllib.load(f)
except Exception:
return {}
@lru_cache(maxsize=1)
def get_hardware_mark_list() -> frozenset[str]:
"""Return hardware SKU marker names from ``pyproject.toml``.
A marker is treated as a hardware resource marker when its description
contains the ``[hardware-resource]`` tag, e.g.::
"H100: [hardware-resource] Tests that require H100 GPU"
"""
data = _load_pyproject()
entries = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("markers", [])
if not entries:
# Fallback when TOML parsing fails: scan tagged marker lines.
text = (_repo_root() / "pyproject.toml").read_text(encoding="utf-8")
return frozenset(
re.findall(
rf'^\s*"([A-Za-z0-9_]+)\s*:\s*{re.escape(_HARDWARE_RESOURCE_MARKER_TAG)}',
text,
flags=re.M,
)
)
names: set[str] = set()
for entry in entries:
text = str(entry)
if _HARDWARE_RESOURCE_MARKER_TAG not in text:
continue
name = text.split(":", 1)[0].strip()
if name:
names.add(name)
return frozenset(names)
def cuda_marks(*, res: str, num_cards: int):