From 4b625d1175e367e313ef9166a5d296d558fcfce4 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:20:04 +0800 Subject: [PATCH] [CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5845) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] Co-authored-by: amy-why-3459 --- .../test_examples/l4_performance_tests.inc.md | 2 +- pyproject.toml | 17 +- tests/dfx/conftest.py | 161 +++++++++---- .../scripts/diffusion_result_template.json | 10 +- .../perf/scripts/result_omni_template.json | 8 +- tests/dfx/perf/scripts/run_benchmark.py | 6 +- .../perf/scripts/run_diffusion_benchmark.py | 20 +- .../test_boogu_image_edit_vllm_omni.json | 4 +- .../tests/test_boogu_image_vllm_omni.json | 4 +- .../perf/tests/test_hunyuan_image3_it2i.json | 2 +- .../tests/test_hunyuan_image_tp2_cfgp2.json | 2 +- .../tests/test_hunyuan_image_tp2_sp2.json | 2 +- .../perf/tests/test_hunyuan_image_tp4.json | 2 +- tests/dfx/perf/tests/test_runner_metadata.py | 219 +++++++++++++++--- tests/helpers/mark.py | 61 ++++- 15 files changed, 414 insertions(+), 106 deletions(-) diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 48d327334..8ff27ee08 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -200,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par | num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | | request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | | max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | -| baseline | object | No | see below | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS. +| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. | diff --git a/pyproject.toml b/pyproject.toml index 38936daaa..f8328e656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -250,13 +250,16 @@ markers = [ "npu: Tests that run on NPU/Ascend (auto-added)", "musa: Tests that run on MUSA/Moore Threads (auto-added)", # specified computation resources marks (auto-added) - "H100: Tests that require H100 GPU", - "L4: Tests that require L4 GPU", - "MI325: Tests that require MI325 GPU (AMD/ROCm)", - "B60: Tests that require Intel Arc Pro B60 XPU", - "S5000: Tests that require S5000 GPU (Moore Threads/MUSA)", - "A2: Tests that require A2 NPU", - "A3: Tests that require A3 NPU", + # Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list(). + "H100: [hardware-resource] Tests that require H100 GPU", + "H200: [hardware-resource] Tests that require H200 GPU", + "L4: [hardware-resource] Tests that require L4 GPU", + "B200: [hardware-resource] Tests that require B200 GPU", + "MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)", + "B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU", + "S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)", + "A2: [hardware-resource] Tests that require A2 NPU", + "A3: [hardware-resource] Tests that require A3 NPU", "distributed_cuda: Tests that require multi cards on CUDA platform", "distributed_rocm: Tests that require multi cards on ROCm platform", "distributed_xpu: Tests that require multi cards on XPU platform", diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index dbd6cc2d0..f81dee11c 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -1,4 +1,3 @@ -import copy import json import os import re @@ -10,7 +9,7 @@ from typing import Any import pytest -from tests.helpers.mark import hardware_marks +from tests.helpers.mark import get_hardware_mark_list, hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config from vllm_omni.platforms import current_omni_platform @@ -447,36 +446,16 @@ def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> li return unique -def extract_mark_resource_label(mark_field: Any) -> str: - """Return a filename-safe hardware label from ``mark.hardware_marks.res`` values. - - Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``. - - Prefer :func:`get_runtime_resource_label` for perf result filenames so labels - reflect the machine that actually ran the benchmark. - """ - if isinstance(mark_field, list): - for item in mark_field: - if isinstance(item, dict) and "hardware_marks" in item: - return extract_mark_resource_label(item) - return "na" - if not isinstance(mark_field, dict): - return "na" - hw = mark_field.get("hardware_marks") - if not isinstance(hw, dict): - return "na" - res = hw.get("res") - if not isinstance(res, dict) or not res: - return "na" - labels = [_safe_filename_token(value) for value in res.values()] - return "-".join(labels) if labels else "na" - - -_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = ( +# Device marketing-name tokens for ``get_runtime_resource_label`` only. +# Longer / more specific tokens must appear before shorter substrings +# (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``). +_RUNTIME_DEVICE_ALIASES: tuple[str, ...] = ( "H100", "H800", "H200", "H20", + "B200", + "GB200", "L40S", "L40", "L4", @@ -505,7 +484,7 @@ def _normalize_runtime_device_label(raw: str) -> str: if not raw or not str(raw).strip(): return "na" upper = str(raw).upper() - for token in _KNOWN_RUNTIME_RESOURCE_TOKENS: + for token in _RUNTIME_DEVICE_ALIASES: if token.upper() in upper: return _safe_filename_token(token) compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw)) @@ -558,10 +537,108 @@ def resource_label_for_filename(resource_label: str | None) -> str: return token -def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str: - """Return runtime hardware label for perf result filenames.""" - del configs - return get_runtime_resource_label() +def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool: + """True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines. + + Detection is structural + hardware-label allowlist (not metric-name hints): + + - every top-level value is a non-empty dict (per-hardware metric map); + - every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list` + (``[hardware-resource]`` markers from ``pyproject.toml``). + + Metric names under each hardware bucket are unconstrained. + Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated. + """ + if not baseline: + return False + if not all(isinstance(value, dict) and bool(value) for value in baseline.values()): + return False + allowed = {label.upper() for label in get_hardware_mark_list()} + return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys()) + + +def resolve_baseline_value( + baseline_raw: Any, + *, + sweep_index: int | None, +) -> Any: + """Pick the baseline threshold for one metric at this sweep step. + + Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar. + """ + if baseline_raw is None: + return None + if isinstance(baseline_raw, dict): + raise TypeError( + "per-metric baseline dict keyed by concurrency/request-rate is not supported; " + "use a sweep-aligned list or a scalar under each hardware bucket" + ) + if isinstance(baseline_raw, (list, tuple)): + if sweep_index is None: + raise ValueError("list baseline requires sweep_index") + if not (0 <= sweep_index < len(baseline_raw)): + raise IndexError(f"baseline list len={len(baseline_raw)} has no index {sweep_index}") + return baseline_raw[sweep_index] + return baseline_raw + + +def resolve_baseline_for_sweep( + baseline_config: dict[str, Any] | None, + *, + sweep_index: int | None = None, +) -> dict[str, Any]: + """Resolve sweep-dependent baselines while keeping every hardware bucket. + + Expects hardware-nested input only:: + + {"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}} + + with ``sweep_index=2`` (e.g. concurrency 32) becomes:: + + {"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}} + + Per-metric values may be a sweep-aligned list or a scalar. + """ + if not baseline_config: + return {} + if not is_hardware_nested_baseline(baseline_config): + allowed = sorted(get_hardware_mark_list()) + unknown = [ + str(key).strip() + for key in baseline_config.keys() + if str(key).strip().upper() not in {label.upper() for label in allowed} + ] + nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values()) + if nested_shape and unknown: + raise ValueError( + "baseline top-level keys must be [hardware-resource] pytest markers from " + "pyproject.toml (tool.pytest.ini_options.markers). " + f"Unknown hardware label(s): {unknown!r}. " + f"Known markers: {allowed!r}. " + "Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' " + "to pyproject.toml, then retry." + ) + raise ValueError( + "baseline must be hardware-nested with [hardware-resource] marker names as " + "top-level keys, e.g. " + '{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. ' + f"got top-level keys={list(baseline_config.keys())!r}; " + f"known markers={allowed!r}. " + "If you need a new hardware bucket, add it to pyproject.toml markers with " + "the [hardware-resource] tag." + ) + + return { + hardware: { + metric_name: resolve_baseline_value( + baseline_raw, + sweep_index=sweep_index, + ) + for metric_name, baseline_raw in metrics.items() + } + for hardware, metrics in baseline_config.items() + if isinstance(metrics, dict) + } def run_benchmark( @@ -572,17 +649,18 @@ def run_benchmark( num_prompt: int, *, baseline_config: dict[str, Any] | None = None, + sweep_index: int | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, resource_label: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. - After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the raw - ``baseline`` object from the test config (when ``baseline_config`` is provided), - including any hardware-nested map (e.g. ``{"H100": {...}}``). Downstream report - tooling can select a hardware bucket when needed. If the benchmark exits without - writing a result file, ``result_omni_template.json`` is used as a fallback. + After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the + sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested + maps keep every hardware bucket, but each metric is reduced to the value for + this concurrency / request-rate step. If the benchmark exits without writing a + result file, ``result_omni_template.json`` is used as a fallback. """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) @@ -649,8 +727,11 @@ def run_benchmark( result = json.load(f) if baseline_config: - # Persist the JSON baseline as-is (no hardware / sweep resolution). - result["baseline"] = copy.deepcopy(baseline_config) + # Keep every hardware bucket; resolve list metrics to this sweep step. + result["baseline"] = resolve_baseline_for_sweep( + baseline_config, + sweep_index=sweep_index, + ) else: result["baseline"] = {} if random_input_len is not None: diff --git a/tests/dfx/perf/scripts/diffusion_result_template.json b/tests/dfx/perf/scripts/diffusion_result_template.json index ba16a993d..abb41c48b 100644 --- a/tests/dfx/perf/scripts/diffusion_result_template.json +++ b/tests/dfx/perf/scripts/diffusion_result_template.json @@ -21,10 +21,12 @@ "num-input-images": 0, "enable-negative-prompt": false, "baseline": { - "throughput_qps": 0, - "latency_mean": 0, - "peak_memory_mb_max": 0, - "peak_memory_mb_mean": 0 + "H100": { + "throughput_qps": 0, + "latency_mean": 0, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0 + } } }, "result": { diff --git a/tests/dfx/perf/scripts/result_omni_template.json b/tests/dfx/perf/scripts/result_omni_template.json index 1d6132140..ca94be0f7 100644 --- a/tests/dfx/perf/scripts/result_omni_template.json +++ b/tests/dfx/perf/scripts/result_omni_template.json @@ -46,9 +46,11 @@ "median_audio_duration_s": 0, "p99_audio_duration_s": 0, "baseline": { - "mean_ttft_ms": 0, - "mean_audio_ttfp_ms": 0, - "mean_audio_rtf": 0 + "H100": { + "mean_ttft_ms": 0, + "mean_audio_ttfp_ms": 0, + "mean_audio_rtf": 0 + } }, "random_input_len": 0, "random_output_len": 0 diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 2c414a961..4ab7959d2 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -237,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params): break # QPS / request-rate sweep - for qps, num_prompt in zip(qps_list, num_prompt_list): + for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] result = run_benchmark( args=args, @@ -246,6 +246,7 @@ def test_performance_benchmark(omni_server, benchmark_params): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), + sweep_index=sweep_index, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, @@ -253,7 +254,7 @@ def test_performance_benchmark(omni_server, benchmark_params): assert_result(result, num_prompt) # concurrency test - for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list): + for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] result = run_benchmark( args=args, @@ -262,6 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), + sweep_index=sweep_index, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 43aff27f5..92f85587c 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -23,7 +23,6 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u the same per-file aggregation; ``-m`` only selects which cases run. """ -import copy import json import os import socket @@ -45,6 +44,7 @@ from tests.dfx.conftest import ( get_runtime_resource_label, hardware_json_value, is_diffusion_perf_config, + resolve_baseline_for_sweep, resolve_pytest_marks, resource_label_for_filename, ) @@ -766,8 +766,8 @@ def run_benchmark( completed = metrics.get("completed_requests", metrics.get("completed", 0)) failed = metrics.get("failed_requests", metrics.get("failed", 0)) - # Persist the JSON baseline as-is (no hardware / sweep resolution). - baseline = copy.deepcopy(params.get("baseline") or {}) + # Persist sweep-resolved baseline from params (already narrowed in _build_run_params). + baseline = params.get("baseline") or {} metrics["baseline"] = baseline record: dict[str, Any] = { @@ -862,6 +862,7 @@ def _build_run_params( params: dict[str, Any], *, num_prompts: int, + sweep_index: int | None = None, request_rate: Any | None = None, max_concurrency: Any | None = None, ) -> dict[str, Any]: @@ -873,6 +874,12 @@ def _build_run_params( run_params["request-rate"] = request_rate if max_concurrency is not None: run_params["max-concurrency"] = max_concurrency + if "baseline" in params: + # Keep all hardware buckets; pick the metric value for this sweep step only. + run_params["baseline"] = resolve_baseline_for_sweep( + params.get("baseline"), + sweep_index=sweep_index, + ) return run_params @@ -895,19 +902,20 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: sweep_runs: list[dict[str, Any]] = [] - for request_rate, num_prompts in zip(request_rate_list, num_prompt_list): + for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)): sweep_runs.append( { "params": _build_run_params( params, request_rate=request_rate, num_prompts=num_prompts, + sweep_index=sweep_index, ), "num_prompts": num_prompts, } ) - for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list): + for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)): sweep_runs.append( { "params": _build_run_params( @@ -915,6 +923,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: max_concurrency=max_concurrency, num_prompts=num_prompts, request_rate="inf", + sweep_index=sweep_index, ), "num_prompts": num_prompts, } @@ -927,6 +936,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: "params": _build_run_params( params, num_prompts=default_num_prompts, + sweep_index=0, ), "num_prompts": default_num_prompts, } diff --git a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json index 21f2caf87..e2f96f626 100644 --- a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json @@ -25,8 +25,8 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302}, - "latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7}, + "throughput_qps": [0.0303, 0.0302, 0.0302], + "latency_mean": [32.6, 62.2, 111.7], "peak_memory_mb_max": 40500, "peak_memory_mb_mean": 40500 } diff --git a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json index 3df89290f..22218268e 100644 --- a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json @@ -25,8 +25,8 @@ "enable-negative-prompt": true, "baseline": { "H100": { - "throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549}, - "latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4}, + "throughput_qps": [0.0549, 0.0547, 0.0549], + "latency_mean": [18.0, 34.3, 61.4], "peak_memory_mb_max": 40500, "peak_memory_mb_mean": 40500 } diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index 33063bcd3..c4ef34b96 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -134,7 +134,7 @@ "max-concurrency": 16, "num-prompts": 96, "baseline": { - "H100": { + "H200": { "throughput_qps": 0.1874, "latency_p99": 101.852, "latency_mean": 79.0385, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index 76a0bb75a..a86a6a3d5 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -38,7 +38,7 @@ "num-prompts": 10, "max-concurrency": 1, "baseline": { - "H100": { + "H200": { "throughput_qps": 0.21, "latency_p99": 4.7469, "peak_memory_mb_max": 101100, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index 70ed93f58..e49cd0543 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -38,7 +38,7 @@ "num-prompts": 10, "max-concurrency": 1, "baseline": { - "H100": { + "H200": { "throughput_qps": 0.20, "latency_p99": 5.1025, "peak_memory_mb_max": 97402, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index 6157c2f1d..54631471a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -37,7 +37,7 @@ "num-prompts": 10, "max-concurrency": 1, "baseline": { - "H100": { + "H200": { "throughput_qps": 0.213, "latency_mean": 4.6953, "latency_p99": 4.8601, diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index d6d1e0b30..81035a272 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -17,7 +17,7 @@ def test_task_excluded_from_cli_args(): "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "percentile-metrics": "audio_rtf,audio_ttfp", - "baseline": {"mean_audio_rtf": [0.5]}, + "baseline": {"H100": {"mean_audio_rtf": [0.5]}}, } exclude_keys = { "request_rate", @@ -123,35 +123,6 @@ def test_resolve_pytest_marks_rejects_legacy_object_format(): ) -def test_extract_mark_resource_label(): - from tests.dfx.conftest import extract_mark_resource_label - - assert extract_mark_resource_label(None) == "na" - assert ( - extract_mark_resource_label( - [ - {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, - "full_model", - ] - ) - == "H100" - ) - assert ( - extract_mark_resource_label( - [ - { - "hardware_marks": { - "res": {"cuda": "H100", "rocm": "MI325"}, - "num_cards": 2, - } - }, - "diffusion", - ] - ) - == "H100-MI325" - ) - - def test_resource_label_for_filename(): from tests.dfx.conftest import resource_label_for_filename @@ -169,16 +140,14 @@ def test_hardware_json_value(): assert hardware_json_value(None) == "" -def test_extract_configs_resource_label(monkeypatch): - from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label +def test_get_runtime_resource_label(monkeypatch): + from tests.dfx.conftest import get_runtime_resource_label monkeypatch.setattr( "tests.dfx.conftest._read_runtime_device_name", lambda *, device_id=0: "NVIDIA H100 80GB HBM3", ) - get_runtime_resource_label(refresh=True) - assert extract_configs_resource_label([]) == "H100" - get_runtime_resource_label(refresh=True) + assert get_runtime_resource_label(refresh=True) == "H100" monkeypatch.setattr( "tests.dfx.conftest._read_runtime_device_name", lambda *, device_id=0: "Ascend910B2", @@ -336,3 +305,183 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat ("start", tts_server), ("stop", tts_server), ] + + +def test_is_hardware_nested_baseline(): + from tests.dfx.conftest import ( + _RUNTIME_DEVICE_ALIASES, + is_hardware_nested_baseline, + ) + from tests.helpers.mark import get_hardware_mark_list + + hardware_marks = get_hardware_mark_list() + assert hardware_marks + assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks + assert "cuda" not in hardware_marks + assert "full_model" not in hardware_marks + + # Runtime aliases are a full static list (independent of baseline allowlist). + assert "H100" in _RUNTIME_DEVICE_ALIASES + assert "A100" in _RUNTIME_DEVICE_ALIASES + + assert is_hardware_nested_baseline( + { + "H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]}, + "A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]}, + } + ) + # Custom metric names under a known hardware label are allowed. + assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}}) + assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}}) + # Flat metric maps / concurrency-keyed maps are not hardware-nested. + assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]}) + assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}}) + assert not is_hardware_nested_baseline({}) + # Alias-only labels (not [hardware-resource] markers) are rejected for baselines. + assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}}) + # Unknown top-level hardware label is rejected. + assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}}) + # Empty per-hardware metric map is rejected. + assert not is_hardware_nested_baseline({"H100": {}}) + + +def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + baseline = { + "H100": { + "mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8], + "mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0], + }, + "A3": { + "mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0], + "mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0], + }, + } + # max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32 + got = resolve_baseline_for_sweep(baseline, sweep_index=4) + assert got == { + "H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0}, + "A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0}, + } + # First sweep step keeps both hardware buckets too. + got0 = resolve_baseline_for_sweep(baseline, sweep_index=0) + assert set(got0) == {"H100", "A3"} + assert got0["H100"]["mean_ttft_ms"] == 96.4 + assert got0["A3"]["mean_ttft_ms"] == 90.0 + + +def test_resolve_baseline_for_sweep_rejects_flat_baseline(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"): + resolve_baseline_for_sweep( + {"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]}, + sweep_index=1, + ) + + +def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"): + resolve_baseline_for_sweep( + {"A100": {"throughput_qps": 1.0}}, + sweep_index=0, + ) + + +def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + # Sweep-aligned lists under each hardware bucket (canonical form). + listed = { + "H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]}, + "A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]}, + } + assert resolve_baseline_for_sweep(listed, sweep_index=1) == { + "H100": {"throughput_qps": 0.6, "latency_mean": 2.0}, + "A3": {"throughput_qps": 0.7, "latency_mean": 1.8}, + } + + # Scalars under hardware stay as-is (single-concurrency cases). + scalar = {"H100": {"throughput_qps": 0.5}} + assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}} + # Custom metric names are preserved. + custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}} + assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}} + assert resolve_baseline_for_sweep(None) == {} + assert resolve_baseline_for_sweep({}) == {} + + +def test_resolve_baseline_value_errors(): + from tests.dfx.conftest import resolve_baseline_value + + with pytest.raises(ValueError, match="sweep_index"): + resolve_baseline_value([1.0, 2.0], sweep_index=None) + with pytest.raises(IndexError): + resolve_baseline_value([1.0], sweep_index=1) + with pytest.raises(TypeError, match="not supported"): + resolve_baseline_value({"1": 0.4}, sweep_index=0) + + +def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch): + """``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency.""" + import importlib + import sys + + cfg = tmp_path / "mini_diffusion_perf.json" + cfg.write_text( + json.dumps( + [ + { + "test_name": "test_mini", + "server_type": "vllm-omni", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "diffusion", + "full_model", + ], + "server_params": {"model": "m/mini"}, + "benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}], + } + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)]) + sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None) + from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb + + importlib.reload(rdb) + + params = { + "name": "c_sweep", + "dataset": "random", + "task": "t2i", + "num-prompts": [8, 16, 32], + "max-concurrency": [1, 8, 32], + "baseline": { + "H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]}, + "A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]}, + }, + } + run32 = rdb._build_run_params( + params, + num_prompts=32, + max_concurrency=32, + request_rate="inf", + sweep_index=2, + ) + assert run32["max-concurrency"] == 32 + assert run32["baseline"] == { + "H100": {"throughput_qps": 0.3, "latency_mean": 30.0}, + "A3": {"throughput_qps": 0.33, "latency_mean": 29.0}, + } + + sweeps = rdb._iter_sweep_runs(params) + assert len(sweeps) == 3 + assert sweeps[2]["params"]["max-concurrency"] == 32 + assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3 + assert "A3" in sweeps[2]["params"]["baseline"] + assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list) diff --git a/tests/helpers/mark.py b/tests/helpers/mark.py index 3475a0012..676732a8e 100644 --- a/tests/helpers/mark.py +++ b/tests/helpers/mark.py @@ -2,10 +2,69 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).""" +from __future__ import annotations + +import re +from functools import lru_cache +from pathlib import Path + import pytest from vllm.platforms import current_platform -# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor). +# Marker description tag in ``pyproject.toml`` ``tool.pytest.ini_options.markers``. +# Example: ``"H100: [hardware-resource] Tests that require H100 GPU"``. +_HARDWARE_RESOURCE_MARKER_TAG = "[hardware-resource]" + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _load_pyproject() -> dict: + path = _repo_root() / "pyproject.toml" + try: + try: + import tomllib + except ModuleNotFoundError: # Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + + with path.open("rb") as f: + return tomllib.load(f) + except Exception: + return {} + + +@lru_cache(maxsize=1) +def get_hardware_mark_list() -> frozenset[str]: + """Return hardware SKU marker names from ``pyproject.toml``. + + A marker is treated as a hardware resource marker when its description + contains the ``[hardware-resource]`` tag, e.g.:: + + "H100: [hardware-resource] Tests that require H100 GPU" + """ + data = _load_pyproject() + entries = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("markers", []) + if not entries: + # Fallback when TOML parsing fails: scan tagged marker lines. + text = (_repo_root() / "pyproject.toml").read_text(encoding="utf-8") + return frozenset( + re.findall( + rf'^\s*"([A-Za-z0-9_]+)\s*:\s*{re.escape(_HARDWARE_RESOURCE_MARKER_TAG)}', + text, + flags=re.M, + ) + ) + + names: set[str] = set() + for entry in entries: + text = str(entry) + if _HARDWARE_RESOURCE_MARKER_TAG not in text: + continue + name = text.split(":", 1)[0].strip() + if name: + names.add(name) + return frozenset(names) def cuda_marks(*, res: str, num_cards: int):