From 0a12ac52ee4ec6d9d59b7bfdc80cde9090572819 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:54:38 +0800 Subject: [PATCH] [CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5858) Signed-off-by: wangyu <410167048@qq.com> Co-authored-by: amy-why-3459 --- .../test_examples/l4_performance_tests.inc.md | 14 +- pyproject.toml | 17 +- recipes/Tencent/HunyuanImage-3.0-Instruct.md | 9 +- tests/dfx/conftest.py | 243 ++++++-------- .../scripts/diffusion_result_template.json | 10 +- .../perf/scripts/result_omni_template.json | 8 +- tests/dfx/perf/scripts/run_benchmark.py | 158 +++++---- .../perf/scripts/run_diffusion_benchmark.py | 134 ++------ .../dfx/perf/tests/test_bagel_vllm_omni.json | 40 ++- .../test_boogu_image_edit_vllm_omni.json | 10 +- .../tests/test_boogu_image_vllm_omni.json | 10 +- .../perf/tests/test_cosmos3_vllm_omni.json | 32 +- tests/dfx/perf/tests/test_higgs_audio_v3.json | 34 +- .../perf/tests/test_hunyuan_image3_it2i.json | 12 +- .../tests/test_hunyuan_image_tp2_cfgp2.json | 10 +- .../tests/test_hunyuan_image_tp2_sp2.json | 10 +- .../perf/tests/test_hunyuan_image_tp4.json | 12 +- .../tests/test_lingbot_video_vllm_omni.json | 3 +- tests/dfx/perf/tests/test_minicpmo_4_5.json | 56 ++-- .../test_minicpmo_4_5_duplex_seed_tts.json | 34 +- .../tests/test_qwen3_omni_async_chunk.json | 179 +++++----- .../tests/test_qwen3_omni_multi_replicas.json | 34 +- .../tests/test_qwen3_omni_no_async_chunk.json | 107 +++--- .../perf/tests/test_qwen3_omni_vllm_text.json | 72 ++-- .../test_qwen_image_edit_2511_vllm_omni.json | 53 +-- .../tests/test_qwen_image_edit_vllm_omni.json | 50 +-- .../test_qwen_image_layered_vllm_omni.json | 20 +- .../perf/tests/test_qwen_image_vllm_omni.json | 64 ++-- tests/dfx/perf/tests/test_runner_metadata.py | 310 +++++++++++++----- tests/dfx/perf/tests/test_tts.json | 248 +++++++------- tests/dfx/perf/tests/test_voxcpm2.json | 104 +++--- .../perf/tests/test_wan22_i2v_vllm_omni.json | 27 +- tests/helpers/mark.py | 61 +++- 33 files changed, 1207 insertions(+), 978 deletions(-) diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 9bdcd29fe..8ff27ee08 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -44,9 +44,11 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [500, 800], - "mean_audio_ttfp_ms": [2000, 3500], - "mean_audio_rtf": [0.25, 0.35] + "H100": { + "mean_ttft_ms": [500, 800], + "mean_audio_ttfp_ms": [2000, 3500], + "mean_audio_rtf": [0.25, 0.35] + } } } ] @@ -142,10 +144,6 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json - -# Optional baseline assertion (default off) -pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \ - --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json ``` See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). @@ -202,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par | num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | | request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | | max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | -| baseline | object | No | see above | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string). +| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. | diff --git a/pyproject.toml b/pyproject.toml index c442c3164..4ef963970 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -243,13 +243,16 @@ markers = [ "npu: Tests that run on NPU/Ascend (auto-added)", "musa: Tests that run on MUSA/Moore Threads (auto-added)", # specified computation resources marks (auto-added) - "H100: Tests that require H100 GPU", - "L4: Tests that require L4 GPU", - "MI325: Tests that require MI325 GPU (AMD/ROCm)", - "B60: Tests that require Intel Arc Pro B60 XPU", - "S5000: Tests that require S5000 GPU (Moore Threads/MUSA)", - "A2: Tests that require A2 NPU", - "A3: Tests that require A3 NPU", + # Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list(). + "H100: [hardware-resource] Tests that require H100 GPU", + "H200: [hardware-resource] Tests that require H200 GPU", + "L4: [hardware-resource] Tests that require L4 GPU", + "B200: [hardware-resource] Tests that require B200 GPU", + "MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)", + "B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU", + "S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)", + "A2: [hardware-resource] Tests that require A2 NPU", + "A3: [hardware-resource] Tests that require A3 NPU", "distributed_cuda: Tests that require multi cards on CUDA platform", "distributed_rocm: Tests that require multi cards on ROCm platform", "distributed_xpu: Tests that require multi cards on XPU platform", diff --git a/recipes/Tencent/HunyuanImage-3.0-Instruct.md b/recipes/Tencent/HunyuanImage-3.0-Instruct.md index f169e6a6b..a9a63d5ce 100644 --- a/recipes/Tencent/HunyuanImage-3.0-Instruct.md +++ b/recipes/Tencent/HunyuanImage-3.0-Instruct.md @@ -164,10 +164,11 @@ curl -s http://localhost:8091/v1/chat/completions \ PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds performance CI configs for the same DiT-only settings. The CI step is currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail` -enabled, intended for initial data collection. Performance assertions are -skipped (`skip-performance-assertion: true`); the baseline values in the -JSON configs are reference-only and will be promoted to regression gates -once enough nightly data has been collected. +enabled, intended for initial data collection. Perf runners no longer +compare metrics against JSON baselines at assert time; the `baseline` +blocks in the configs are reference-only and are copied into result JSON +for reporting. Regression gates can be reintroduced once enough nightly +data has been collected. The user-facing equivalent is to launch one of the CLI commands above and generate 1024x1024 images with 50 denoising steps. diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 2eb88bbd6..f81dee11c 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -3,15 +3,13 @@ import os import re import subprocess import threading -import time from datetime import datetime from pathlib import Path from typing import Any import pytest -from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw -from tests.helpers.mark import hardware_marks +from tests.helpers.mark import get_hardware_mark_list, hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config from vllm_omni.platforms import current_omni_platform @@ -119,32 +117,6 @@ def create_unique_server_pytest_params( ] -def create_benchmark_pytest_params( - benchmark_configs: list[dict[str, Any]], - server_to_benchmark_mapping: dict[str, dict], -) -> list[Any]: - """Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks.""" - marks_by_name = _marks_by_test_name(benchmark_configs) - params: list[Any] = [] - seen: set[str] = set() - for config in benchmark_configs: - test_name = config["test_name"] - if test_name in seen: - continue - seen.add(test_name) - params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) - id_suffixes = _unique_benchmark_param_id_suffixes(params_list) - for idx, id_suffix in enumerate(id_suffixes): - params.append( - pytest.param( - (test_name, idx), - marks=marks_by_name.get(test_name, []), - id=f"{test_name}-{id_suffix}", - ) - ) - return params - - def create_paired_benchmark_pytest_params( server_entries: list[tuple[Any, str]], params_by_test_name: dict[str, list[Any]], @@ -357,10 +329,6 @@ def supports_video_generation(model_name: str) -> bool: return any(key in lower for key in ("wan", "video", "i2v", "t2v")) -def supports_chat_generation(model_name: str) -> bool: - return not supports_video_generation(model_name) - - def parse_stage_devices(stage_config_path: str) -> str: text = Path(stage_config_path).read_text(encoding="utf-8") raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE) @@ -389,48 +357,6 @@ def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> N assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}" -def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None: - """Poll a minimal chat request until success.""" - deadline = time.time() + timeout_sec - payload = json.dumps( - { - "model": model, - "messages": [{"role": "user", "content": "Say hello in one short sentence."}], - "stream": False, - "modalities": ["text"], - } - ) - last_error: str | None = None - while time.time() < deadline: - try: - status, body = post_chat_completions_raw(host, port, payload) - if status == 200: - return - last_error = f"http={status} body={body[:200]!r}" - except Exception as exc: # noqa: BLE001 - last_error = str(exc) - time.sleep(2) - raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}") - - -def assert_no_extra_worker_processes( - baseline_pids: set[int], - worker_pattern: str, - timeout_sec: int = 60, -) -> None: - """Ensure no extra worker PIDs remain after teardown.""" - deadline = time.time() + timeout_sec - while time.time() < deadline: - current = set(list_remote_process_pids_by_pattern(worker_pattern)) - extra = current - baseline_pids - if not extra: - return - time.sleep(2) - current = set(list_remote_process_pids_by_pattern(worker_pattern)) - extra = sorted(current - baseline_pids) - assert not extra, f"orphan worker processes remain after container teardown: {extra}" - - def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]: mapping = {} for config in configs: @@ -520,36 +446,16 @@ def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> li return unique -def extract_mark_resource_label(mark_field: Any) -> str: - """Return a filename-safe hardware label from ``mark.hardware_marks.res`` values. - - Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``. - - Prefer :func:`get_runtime_resource_label` for perf result filenames so labels - reflect the machine that actually ran the benchmark. - """ - if isinstance(mark_field, list): - for item in mark_field: - if isinstance(item, dict) and "hardware_marks" in item: - return extract_mark_resource_label(item) - return "na" - if not isinstance(mark_field, dict): - return "na" - hw = mark_field.get("hardware_marks") - if not isinstance(hw, dict): - return "na" - res = hw.get("res") - if not isinstance(res, dict) or not res: - return "na" - labels = [_safe_filename_token(value) for value in res.values()] - return "-".join(labels) if labels else "na" - - -_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = ( +# Device marketing-name tokens for ``get_runtime_resource_label`` only. +# Longer / more specific tokens must appear before shorter substrings +# (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``). +_RUNTIME_DEVICE_ALIASES: tuple[str, ...] = ( "H100", "H800", "H200", "H20", + "B200", + "GB200", "L40S", "L40", "L4", @@ -578,7 +484,7 @@ def _normalize_runtime_device_label(raw: str) -> str: if not raw or not str(raw).strip(): return "na" upper = str(raw).upper() - for token in _KNOWN_RUNTIME_RESOURCE_TOKENS: + for token in _RUNTIME_DEVICE_ALIASES: if token.upper() in upper: return _safe_filename_token(token) compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw)) @@ -631,34 +537,41 @@ def resource_label_for_filename(resource_label: str | None) -> str: return token -def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str: - """Return runtime hardware label for perf result filenames.""" - del configs - return get_runtime_resource_label() +def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool: + """True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines. + + Detection is structural + hardware-label allowlist (not metric-name hints): + + - every top-level value is a non-empty dict (per-hardware metric map); + - every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list` + (``[hardware-resource]`` markers from ``pyproject.toml``). + + Metric names under each hardware bucket are unconstrained. + Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated. + """ + if not baseline: + return False + if not all(isinstance(value, dict) and bool(value) for value in baseline.values()): + return False + allowed = {label.upper() for label in get_hardware_mark_list()} + return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys()) def resolve_baseline_value( baseline_raw: Any, *, sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, ) -> Any: - """Pick the baseline threshold for this sweep step.""" + """Pick the baseline threshold for one metric at this sweep step. + + Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar. + """ if baseline_raw is None: - return 100000 + return None if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" + raise TypeError( + "per-metric baseline dict keyed by concurrency/request-rate is not supported; " + "use a sweep-aligned list or a scalar under each hardware bucket" ) if isinstance(baseline_raw, (list, tuple)): if sweep_index is None: @@ -669,22 +582,62 @@ def resolve_baseline_value( return baseline_raw -def _baseline_thresholds_for_step( - baseline_data: dict[str, Any], +def resolve_baseline_for_sweep( + baseline_config: dict[str, Any] | None, *, sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, ) -> dict[str, Any]: - """Resolve baseline config to one threshold per metric for this iteration.""" - return { - metric_name: resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, + """Resolve sweep-dependent baselines while keeping every hardware bucket. + + Expects hardware-nested input only:: + + {"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}} + + with ``sweep_index=2`` (e.g. concurrency 32) becomes:: + + {"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}} + + Per-metric values may be a sweep-aligned list or a scalar. + """ + if not baseline_config: + return {} + if not is_hardware_nested_baseline(baseline_config): + allowed = sorted(get_hardware_mark_list()) + unknown = [ + str(key).strip() + for key in baseline_config.keys() + if str(key).strip().upper() not in {label.upper() for label in allowed} + ] + nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values()) + if nested_shape and unknown: + raise ValueError( + "baseline top-level keys must be [hardware-resource] pytest markers from " + "pyproject.toml (tool.pytest.ini_options.markers). " + f"Unknown hardware label(s): {unknown!r}. " + f"Known markers: {allowed!r}. " + "Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' " + "to pyproject.toml, then retry." + ) + raise ValueError( + "baseline must be hardware-nested with [hardware-resource] marker names as " + "top-level keys, e.g. " + '{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. ' + f"got top-level keys={list(baseline_config.keys())!r}; " + f"known markers={allowed!r}. " + "If you need a new hardware bucket, add it to pyproject.toml markers with " + "the [hardware-resource] tag." ) - for metric_name, baseline_raw in baseline_data.items() + + return { + hardware: { + metric_name: resolve_baseline_value( + baseline_raw, + sweep_index=sweep_index, + ) + for metric_name, baseline_raw in metrics.items() + } + for hardware, metrics in baseline_config.items() + if isinstance(metrics, dict) } @@ -697,17 +650,17 @@ def run_benchmark( *, baseline_config: dict[str, Any] | None = None, sweep_index: int | None = None, - request_rate: Any | None = None, - max_concurrency: Any | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, resource_label: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. - After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds - (when ``baseline_config`` is provided). If the benchmark exits without writing a result file, - ``result_omni_template.json`` is used as a fallback. + After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the + sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested + maps keep every hardware bucket, but each metric is reduced to the value for + this concurrency / request-rate step. If the benchmark exits without writing a + result file, ``result_omni_template.json`` is used as a fallback. """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) @@ -774,11 +727,10 @@ def run_benchmark( result = json.load(f) if baseline_config: - result["baseline"] = _baseline_thresholds_for_step( + # Keep every hardware bucket; resolve list metrics to this sweep step. + result["baseline"] = resolve_baseline_for_sweep( baseline_config, sweep_index=sweep_index, - request_rate=request_rate, - max_concurrency=max_concurrency, ) else: result["baseline"] = {} @@ -800,12 +752,3 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"), ) - parser.addoption( - "--assert-baseline", - action="store_true", - default=False, - help=( - "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config " - "(default: off)." - ), - ) diff --git a/tests/dfx/perf/scripts/diffusion_result_template.json b/tests/dfx/perf/scripts/diffusion_result_template.json index ba16a993d..abb41c48b 100644 --- a/tests/dfx/perf/scripts/diffusion_result_template.json +++ b/tests/dfx/perf/scripts/diffusion_result_template.json @@ -21,10 +21,12 @@ "num-input-images": 0, "enable-negative-prompt": false, "baseline": { - "throughput_qps": 0, - "latency_mean": 0, - "peak_memory_mb_max": 0, - "peak_memory_mb_mean": 0 + "H100": { + "throughput_qps": 0, + "latency_mean": 0, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0 + } } }, "result": { diff --git a/tests/dfx/perf/scripts/result_omni_template.json b/tests/dfx/perf/scripts/result_omni_template.json index 1d6132140..ca94be0f7 100644 --- a/tests/dfx/perf/scripts/result_omni_template.json +++ b/tests/dfx/perf/scripts/result_omni_template.json @@ -46,9 +46,11 @@ "median_audio_duration_s": 0, "p99_audio_duration_s": 0, "baseline": { - "mean_ttft_ms": 0, - "mean_audio_ttfp_ms": 0, - "mean_audio_rtf": 0 + "H100": { + "mean_ttft_ms": 0, + "mean_audio_ttfp_ms": 0, + "mean_audio_rtf": 0 + } }, "random_input_len": 0, "random_output_len": 0 diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 5794ac10b..03dfac1a0 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -1,6 +1,8 @@ import json import os import threading +from collections.abc import Callable +from contextlib import AbstractContextManager, ExitStack, contextmanager from pathlib import Path from typing import Any @@ -13,15 +15,10 @@ from tests.dfx.conftest import ( get_runtime_resource_label, is_diffusion_perf_config, load_benchmark_configs, - resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer -# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` -# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are -# defined in the same module. -# # Optional JSON field ``mark`` is applied as pytest marks via # ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks": # {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``). @@ -63,35 +60,78 @@ paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_C _omni_server_lock = threading.Lock() +class _SingleActiveContext: + """Reuse one active context while its configuration key is unchanged.""" + + def __init__(self) -> None: + self._key: Any = None + self._stack: ExitStack | None = None + self._value: Any = None + + def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any: + if self._stack is not None and key == self._key: + return self._value + + self.close() + stack = ExitStack() + value = stack.enter_context(factory()) + self._key = key + self._stack = stack + self._value = value + return value + + def close(self) -> None: + stack = self._stack + self._key = None + self._stack = None + self._value = None + if stack is not None: + stack.close() + + +@contextmanager +def _start_omni_server(server_param): + test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param + + print(f"Starting OmniServer with test: {test_name}, model: {model}") + + server_args: list[str] = [] + if use_omni: + server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"] + # --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py): + # deploy-config sets the base; stage-overrides are applied on top. Both can be set. + if stage_config_path: + server_args = ["--deploy-config", stage_config_path] + server_args + if stage_overrides: + server_args = ["--stage-overrides", stage_overrides] + server_args + if extra_cli_args: + server_args = list(extra_cli_args) + server_args + with OmniServer(model, server_args, use_omni=use_omni) as server: + server.test_name = test_name + print("OmniServer started successfully") + yield server + print("OmniServer stopping...") + + print("OmniServer stopped") + + @pytest.fixture(scope="module") -def omni_server(request): +def omni_server_context(): """Start vLLM-Omni server as a subprocess with actual model weights. - Uses session scope so the server starts only once for the entire test session. + Reuse it for adjacent benchmark cases with the same server configuration. Multi-stage initialization can take 10-20+ minutes. """ with _omni_server_lock: - test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = request.param + active_context = _SingleActiveContext() + try: + yield active_context + finally: + active_context.close() - print(f"Starting OmniServer with test: {test_name}, model: {model}") - server_args: list[str] = [] - if use_omni: - server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"] - # --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py): - # deploy-config sets the base; stage-overrides are applied on top. Both can be set. - if stage_config_path: - server_args = ["--deploy-config", stage_config_path] + server_args - if stage_overrides: - server_args = ["--stage-overrides", stage_overrides] + server_args - if extra_cli_args: - server_args = list(extra_cli_args) + server_args - with OmniServer(model, server_args, use_omni=use_omni) as server: - server.test_name = test_name - print("OmniServer started successfully") - yield server - print("OmniServer stopping...") - - print("OmniServer stopped") +@pytest.fixture +def omni_server(request, omni_server_context): + return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param)) @pytest.fixture @@ -117,16 +157,7 @@ def benchmark_params(request): } -def assert_result( - result, - params, - num_prompt, - *, - assert_baseline: bool, - sweep_index: int | None = None, - max_concurrency: Any | None = None, - request_rate: Any | None = None, -) -> None: +def assert_result(result, params, num_prompt) -> None: assert result["completed"] == num_prompt, "Request failures exist" expected_audio_turns = params.get("expected_duplex_audio_turns_per_session") if expected_audio_turns is not None: @@ -139,25 +170,6 @@ def assert_result( isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns for metric in session_metrics ), f"Not every duplex session emitted {expected_audio_turns} audio turns" - if not assert_baseline: - return - baseline_data = params.get("baseline", {}) - for metric_name, baseline_raw in baseline_data.items(): - current_value = result[metric_name] - baseline_value = resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - if "throughput" in metric_name: - assert current_value >= baseline_value, ( - f"Throughput test result was below baseline: {metric_name}: {current_value} < {baseline_value}" - ) - else: - assert current_value <= baseline_value, ( - f"Test result exceeded baseline: {metric_name}: {current_value} > {baseline_value}" - ) @pytest.mark.benchmark @@ -166,7 +178,7 @@ def assert_result( paired_benchmark_params, indirect=["omni_server", "benchmark_params"], ) -def test_performance_benchmark(omni_server, benchmark_params, request): +def test_performance_benchmark(omni_server, benchmark_params): test_name = benchmark_params["test_name"] params = benchmark_params["params"] dataset_name = params.get("dataset_name", "") @@ -178,7 +190,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request): print(f"Running benchmark for model: {model}") print(f"Benchmark parameters: {benchmark_params}") - assert_baseline = request.config.getoption("--assert-baseline", default=False) resource_label = get_runtime_resource_label() def to_list(value, default=None): @@ -213,7 +224,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request): "eval_phase", "trust_remote_code", "expected_duplex_audio_turns_per_session", - "baseline_hardware", } for key, value in params.items(): @@ -239,7 +249,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): break # QPS / request-rate sweep - for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): + for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] result = run_benchmark( args=args, @@ -248,24 +258,15 @@ def test_performance_benchmark(omni_server, benchmark_params, request): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), - sweep_index=i, - request_rate=qps, - max_concurrency=None, + sweep_index=sweep_index, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, ) - assert_result( - result, - params, - num_prompt, - assert_baseline=assert_baseline, - sweep_index=i, - request_rate=qps, - ) + assert_result(result, params, num_prompt) # concurrency test - for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): + for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] result = run_benchmark( args=args, @@ -274,18 +275,9 @@ def test_performance_benchmark(omni_server, benchmark_params, request): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), - sweep_index=i, - request_rate=None, - max_concurrency=concurrency, + sweep_index=sweep_index, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, ) - assert_result( - result, - params, - num_prompt, - assert_baseline=assert_baseline, - sweep_index=i, - max_concurrency=concurrency, - ) + assert_result(result, params, num_prompt) diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index d19c0900e..92f85587c 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -13,8 +13,6 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j pytest run_diffusion_benchmark.py -m "diffusion" pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). - Optional JSON field ``mark`` is applied as pytest marks on that case via ``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``). @@ -46,6 +44,7 @@ from tests.dfx.conftest import ( get_runtime_resource_label, hardware_json_value, is_diffusion_perf_config, + resolve_baseline_for_sweep, resolve_pytest_marks, resource_label_for_filename, ) @@ -274,11 +273,6 @@ def _write_result_record(record: dict[str, Any]) -> Path: return target -def _append_to_aggregated_file(record: dict[str, Any]) -> None: - """Backward-compatible wrapper; prefer :func:`_write_result_record`.""" - _write_result_record(record) - - _server_lock = threading.Lock() # --------------------------------------------------------------------------- @@ -355,6 +349,17 @@ def _kill_process_tree(pid: int) -> None: # --------------------------------------------------------------------------- +def _resolve_offline_model(model: str) -> str: + """Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir.""" + import huggingface_hub + + if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + from huggingface_hub import snapshot_download + + return snapshot_download(model, local_files_only=True) + + class DiffusionServer: """Start a vLLM-Omni diffusion model server as a subprocess. @@ -372,7 +377,7 @@ class DiffusionServer: port: int | None = None, ) -> None: self.server_cfg: dict[str, Any] = server_cfg - self.model = server_cfg["model"] + self.model = _resolve_offline_model(server_cfg["model"]) self.serve_args = server_cfg["serve_args"] self.host = "127.0.0.1" self.port = port if port is not None else get_open_port(self.host) @@ -667,7 +672,7 @@ def run_benchmark( with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) - exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"} + exclude_keys = {"baseline", "dataset", "task", "name"} cmd = [ sys.executable, @@ -761,12 +766,17 @@ def run_benchmark( completed = metrics.get("completed_requests", metrics.get("completed", 0)) failed = metrics.get("failed_requests", metrics.get("failed", 0)) + # Persist sweep-resolved baseline from params (already narrowed in _build_run_params). + baseline = params.get("baseline") or {} + metrics["baseline"] = baseline + record: dict[str, Any] = { "test_name": test_name, "endpoint": endpoint, "timestamp": timestamp, "server_params": server_cfg.get("server_params"), "benchmark_params": params, + "baseline": baseline, "result": metrics, "log_file": str(log_file), "Model": model, @@ -816,70 +826,14 @@ def run_benchmark( # --------------------------------------------------------------------------- -def _resolve_baseline_value( - baseline_raw: Any, - *, - sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> Any: - """Pick the baseline threshold for this sweep step.""" - if baseline_raw is None: - return 100000 - if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" - ) - if isinstance(baseline_raw, (list, tuple)): - if sweep_index is None: - raise ValueError("sweep_index is required when baseline is a list or tuple") - return baseline_raw[sweep_index] - return baseline_raw - - def assert_result( result: dict[str, Any], - params: dict[str, Any], num_prompts: int, - *, - sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, - assert_baseline: bool = True, ) -> None: - """Assert that benchmark metrics satisfy the configured baselines.""" + """Assert that the expected number of requests completed.""" completed = result.get("completed_requests", result.get("completed", 0)) assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" - if not assert_baseline: - return - if params.get("skip-performance-assertion", False): - print("Skipping performance assertions.") - return - - for metric, baseline_raw in params.get("baseline", {}).items(): - current = result.get(metric) - assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}" - threshold = _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - if "throughput" in metric: - assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}" - else: - assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}" - def _default_benchmark_endpoint_for_task(task: str) -> str: """Return the default client-side benchmark endpoint for a diffusion task.""" @@ -921,15 +875,11 @@ def _build_run_params( if max_concurrency is not None: run_params["max-concurrency"] = max_concurrency if "baseline" in params: - run_params["baseline"] = { - metric: _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - for metric, baseline_raw in params["baseline"].items() - } + # Keep all hardware buckets; pick the metric value for this sweep step only. + run_params["baseline"] = resolve_baseline_for_sweep( + params.get("baseline"), + sweep_index=sweep_index, + ) return run_params @@ -952,36 +902,30 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: sweep_runs: list[dict[str, Any]] = [] - for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)): + for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)): sweep_runs.append( { "params": _build_run_params( params, request_rate=request_rate, num_prompts=num_prompts, - sweep_index=i, + sweep_index=sweep_index, ), "num_prompts": num_prompts, - "sweep_index": i, - "request_rate": request_rate, - "max_concurrency": None, } ) - for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)): + for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)): sweep_runs.append( { "params": _build_run_params( params, max_concurrency=max_concurrency, num_prompts=num_prompts, - sweep_index=i, request_rate="inf", + sweep_index=sweep_index, ), "num_prompts": num_prompts, - "sweep_index": i, - "request_rate": None, - "max_concurrency": max_concurrency, } ) @@ -992,11 +936,9 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: "params": _build_run_params( params, num_prompts=default_num_prompts, + sweep_index=0, ), "num_prompts": default_num_prompts, - "sweep_index": None, - "request_rate": None, - "max_concurrency": None, } ) @@ -1012,14 +954,12 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: paired_benchmark_params, indirect=["diffusion_server", "benchmark_params"], ) -def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): +def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): """Run the diffusion performance benchmark and verify request completion. One server is started per unique parallel configuration (module scope). For each server, all benchmark parameter sets defined in the config JSON are executed sequentially; metrics are recorded to the aggregated result file. - - Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block. """ test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -1062,14 +1002,4 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}") print("=" * 60) - assert_baseline = request.config.getoption("--assert-baseline", default=False) - - assert_result( - result, - params, - sweep_run["num_prompts"], - sweep_index=sweep_run["sweep_index"], - max_concurrency=sweep_run["max_concurrency"], - request_rate=sweep_run["request_rate"], - assert_baseline=assert_baseline, - ) + assert_result(result, sweep_run["num_prompts"]) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index e6777acfe..9f6ff0241 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -40,10 +40,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.495, - "latency_mean": 2.0238, - "peak_memory_mb_max": 29393.4286, - "peak_memory_mb_mean": 29392.7143 + "H100": { + "throughput_qps": 0.495, + "latency_mean": 2.0238, + "peak_memory_mb_max": 29393.4286, + "peak_memory_mb_mean": 29392.7143 + } } } ] @@ -90,10 +92,12 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.3451, - "latency_mean": 2.9031, - "peak_memory_mb_max": 30480.2857, - "peak_memory_mb_mean": 30479.2857 + "H100": { + "throughput_qps": 0.3451, + "latency_mean": 2.9031, + "peak_memory_mb_max": 30480.2857, + "peak_memory_mb_mean": 30479.2857 + } } } ] @@ -139,10 +143,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.5048, - "latency_mean": 1.987, - "peak_memory_mb_max": 29380.0, - "peak_memory_mb_mean": 29380.0 + "H100": { + "throughput_qps": 0.5048, + "latency_mean": 1.987, + "peak_memory_mb_max": 29380.0, + "peak_memory_mb_mean": 29380.0 + } } } ] @@ -189,10 +195,12 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.149, - "latency_mean": 6.7157, - "peak_memory_mb_max": 30574.0, - "peak_memory_mb_mean": 30574.0 + "H100": { + "throughput_qps": 0.149, + "latency_mean": 6.7157, + "peak_memory_mb_max": 30574.0, + "peak_memory_mb_mean": 30574.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json index 028889af1..e2f96f626 100644 --- a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json @@ -24,10 +24,12 @@ "warmup-num-inference-steps": 28, "enable-negative-prompt": true, "baseline": { - "throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302}, - "latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7}, - "peak_memory_mb_max": 40500, - "peak_memory_mb_mean": 40500 + "H100": { + "throughput_qps": [0.0303, 0.0302, 0.0302], + "latency_mean": [32.6, 62.2, 111.7], + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } } } ] diff --git a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json index 4ee6ca577..22218268e 100644 --- a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json @@ -24,10 +24,12 @@ "warmup-num-inference-steps": 28, "enable-negative-prompt": true, "baseline": { - "throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549}, - "latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4}, - "peak_memory_mb_max": 40500, - "peak_memory_mb_mean": 40500 + "H100": { + "throughput_qps": [0.0549, 0.0547, 0.0549], + "latency_mean": [18.0, 34.3, 61.4], + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } } } ] diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index 9bf31658d..c4d2305e6 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -48,9 +48,11 @@ "guardrails": false }, "baseline": { - "throughput_qps": 1.107, - "latency_mean": 0.8929, - "peak_memory_mb_mean": 80000.0 + "H100": { + "throughput_qps": 1.107, + "latency_mean": 0.8929, + "peak_memory_mb_mean": 80000.0 + } } } ] @@ -115,9 +117,11 @@ "use_duration_template": false }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.6824, - "peak_memory_mb_mean": 30155.4 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 36.6824, + "peak_memory_mb_mean": 30155.4 + } } } ] @@ -181,9 +185,11 @@ "guardrails": false }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.968, - "peak_memory_mb_mean": 30443.6 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 36.968, + "peak_memory_mb_mean": 30443.6 + } } } ] @@ -251,9 +257,11 @@ "condition_video_keep": "first" }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 37.2117, - "peak_memory_mb_mean": 30135.6 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 37.2117, + "peak_memory_mb_mean": 30135.6 + } } } ] diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 04cc88ac2..75e5983b0 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -46,12 +46,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 223.3584 - ], - "median_audio_rtf": [ - 0.4393 - ] + "H100": { + "median_audio_ttfp_ms": [ + 223.3584 + ], + "median_audio_rtf": [ + 0.4393 + ] + } } }, { @@ -70,15 +72,17 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 276.575 - ], - "median_audio_rtf": [ - 0.5073 - ], - "audio_throughput": [ - 27.3728 - ] + "H100": { + "median_audio_ttfp_ms": [ + 276.575 + ], + "median_audio_rtf": [ + 0.5073 + ], + "audio_throughput": [ + 27.3728 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index 1b6902cba..7521d0109 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -140,11 +140,13 @@ "num-prompts": 96, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.1874, - "latency_p99": 101.852, - "latency_mean": 79.0385, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "H200": { + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index 7a699f53a..4a74e2519 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -36,10 +36,12 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.7469, - "peak_memory_mb_max": 101100, - "peak_memory_mb_mean": 101100 + "H200": { + "throughput_qps": 0.21, + "latency_p99": 4.7469, + "peak_memory_mb_max": 101100, + "peak_memory_mb_mean": 101100 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index 521febd2d..03b7947d6 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -36,10 +36,12 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.20, - "latency_p99": 5.1025, - "peak_memory_mb_max": 97402, - "peak_memory_mb_mean": 97402 + "H200": { + "throughput_qps": 0.2, + "latency_p99": 5.1025, + "peak_memory_mb_max": 97402, + "peak_memory_mb_mean": 97402 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index f2d05564b..6b9ba94a6 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -35,11 +35,13 @@ "max-concurrency": 1, "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.213, - "latency_mean": 4.6953, - "latency_p99": 4.8601, - "peak_memory_mb_max": 56912.0, - "peak_memory_mb_mean": 56912.0 + "H200": { + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json index a6751bda1..07403e78f 100644 --- a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json +++ b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json @@ -26,8 +26,7 @@ "warmup-requests": 1, "warmup-concurrency": 1, "warmup-num-inference-steps": 2, - "enable-negative-prompt": true, - "skip-performance-assertion": true + "enable-negative-prompt": true } ] } diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5.json b/tests/dfx/perf/tests/test_minicpmo_4_5.json index 3feaf2e4c..064d6b8d1 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -1,6 +1,6 @@ [ { - "test_name": "test_minicpmo_4_5", + "test_name": "test_minicpmo_4_5_challenge", "mark": [ { "hardware_marks": { @@ -53,37 +53,39 @@ }, "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", "baseline": { - "request_throughput": [ - 0.5383, - 0.6042, - 0.7547 - ], - "mean_ttft_ms": [ - 333.2633, - 533.8656, - 547.3388 - ], - "mean_e2el_ms": [ - 1857.2154, - 6600.5101, - 10450.6811 - ], - "mean_audio_ttfp_ms": [ - 986.4666, - 3411.0754, - 3352.7515 - ], - "mean_audio_rtf": [ - 0.4423, - 1.5734, - 2.3024 - ] + "A3": { + "request_throughput": [ + 0.5383, + 0.6042, + 0.7547 + ], + "mean_ttft_ms": [ + 333.2633, + 533.8656, + 547.3388 + ], + "mean_e2el_ms": [ + 1857.2154, + 6600.5101, + 10450.6811 + ], + "mean_audio_ttfp_ms": [ + 986.4666, + 3411.0754, + 3352.7515 + ], + "mean_audio_rtf": [ + 0.4423, + 1.5734, + 2.3024 + ] + } } } ] }, { - "test_name": "test_minicpmo_4_5_a2", + "test_name": "test_minicpmo_4_5_a2_challenge", "mark": [ { "hardware_marks": { diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index ec38cd545..a9cb45169 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -1,6 +1,6 @@ [ { - "test_name": "test_minicpmo_4_5_duplex_seed_tts", + "test_name": "test_minicpmo_4_5_duplex_seed_tts_challenge", "mark": [ { "hardware_marks": { @@ -44,21 +44,23 @@ }, "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", "baseline": { - "request_throughput": [ - 0.1594 - ], - "mean_ttft_ms": [ - 274.164 - ], - "mean_e2el_ms": [ - 6250.7696 - ], - "mean_audio_ttfp_ms": [ - 994.422 - ], - "mean_audio_rtf": [ - 0.46163 - ] + "A3": { + "request_throughput": [ + 0.1594 + ], + "mean_ttft_ms": [ + 274.164 + ], + "mean_e2el_ms": [ + 6250.7696 + ], + "mean_audio_ttfp_ms": [ + 994.422 + ], + "mean_audio_rtf": [ + 0.46163 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 013436d80..a73abc5fb 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } @@ -43,27 +44,29 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 96.4061, - 140.8147, - 271.9025, - 362.2916, - 507.776 - ], - "mean_e2el_ms": [ - 18507.4582, - 28365.4956, - 31907.9613, - 48161.5829, - 72630.8601 - ], - "mean_audio_rtf": [ - 0.1175, - 0.18, - 0.2359, - 0.3278, - 0.6296 - ] + "H100": { + "mean_ttft_ms": [ + 96.4061, + 140.8147, + 271.9025, + 362.2916, + 507.776 + ], + "mean_e2el_ms": [ + 18507.4582, + 28365.4956, + 31907.9613, + 48161.5829, + 72630.8601 + ], + "mean_audio_rtf": [ + 0.1175, + 0.18, + 0.2359, + 0.3278, + 0.6296 + ] + } } }, { @@ -90,15 +93,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 354.0951 - ], - "mean_e2el_ms": [ - 3952.5615 - ], - "mean_audio_rtf": [ - 0.117 - ] + "H100": { + "mean_ttft_ms": [ + 354.0951 + ], + "mean_e2el_ms": [ + 3952.5615 + ], + "mean_audio_rtf": [ + 0.117 + ] + } } }, { @@ -127,15 +132,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 259.6317 - ], - "mean_e2el_ms": [ - 4253.3009 - ], - "mean_audio_rtf": [ - 0.1448 - ] + "H100": { + "mean_ttft_ms": [ + 259.6317 + ], + "mean_e2el_ms": [ + 4253.3009 + ], + "mean_audio_rtf": [ + 0.1448 + ] + } } }, { @@ -166,15 +173,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 457.4453 - ], - "mean_e2el_ms": [ - 5912.5623 - ], - "mean_audio_rtf": [ - 0.1967 - ] + "H100": { + "mean_ttft_ms": [ + 457.4453 + ], + "mean_e2el_ms": [ + 5912.5623 + ], + "mean_audio_rtf": [ + 0.1967 + ] + } } }, { @@ -205,20 +214,22 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 90.8033, - 202.1779, - 326.5864, - 544.8914, - 961.3929 - ], - "mean_e2el_ms": [ - 4517.3411, - 7857.9711, - 9861.1687, - 13283.3363, - 19541.0736 - ] + "H100": { + "mean_ttft_ms": [ + 90.8033, + 202.1779, + 326.5864, + 544.8914, + 961.3929 + ], + "mean_e2el_ms": [ + 4517.3411, + 7857.9711, + 9861.1687, + 13283.3363, + 19541.0736 + ] + } } }, { @@ -250,12 +261,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 318.048 - ], - "mean_e2el_ms": [ - 807.6114 - ] + "H100": { + "mean_ttft_ms": [ + 318.048 + ], + "mean_e2el_ms": [ + 807.6114 + ] + } } }, { @@ -289,12 +302,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 186.168 - ], - "mean_e2el_ms": [ - 726.1137 - ] + "H100": { + "mean_ttft_ms": [ + 186.168 + ], + "mean_e2el_ms": [ + 726.1137 + ] + } } }, { @@ -330,12 +345,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 375.5282 - ], - "mean_e2el_ms": [ - 1006.1663 - ] + "H100": { + "mean_ttft_ms": [ + 375.5282 + ], + "mean_e2el_ms": [ + 1006.1663 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index 4e54c6f16..a3d8aaea8 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -35,10 +35,12 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], - "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], - "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], - "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] + "H100": { + "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], + "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], + "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], + "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] + } } }, { @@ -61,9 +63,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [340.5122], - "mean_audio_ttfp_ms": [463.0752], - "mean_audio_rtf": [0.1153] + "H100": { + "mean_ttft_ms": [340.5122], + "mean_audio_ttfp_ms": [463.0752], + "mean_audio_rtf": [0.1153] + } } }, { @@ -88,9 +92,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [236.926], - "mean_audio_ttfp_ms": [395.2182], - "mean_audio_rtf": [0.1259] + "H100": { + "mean_ttft_ms": [236.926], + "mean_audio_ttfp_ms": [395.2182], + "mean_audio_rtf": [0.1259] + } } }, { @@ -117,9 +123,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [470.6126], - "mean_audio_ttfp_ms": [665.5474], - "mean_audio_rtf": [0.166] + "H100": { + "mean_ttft_ms": [470.6126], + "mean_audio_ttfp_ms": [665.5474], + "mean_audio_rtf": [0.166] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 71c90f0e7..38dbfb2e3 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } @@ -43,27 +44,29 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 115.8348, - 173.1823, - 446.672, - 464.5985, - 3420.0634 - ], - "mean_e2el_ms": [ - 25070.9523, - 31992.0646, - 37619.7669, - 50393.8257, - 136619.2743 - ], - "mean_audio_rtf": [ - 0.169, - 0.2111, - 0.2475, - 0.3551, - 0.9266 - ] + "H100": { + "mean_ttft_ms": [ + 115.8348, + 173.1823, + 446.672, + 464.5985, + 3420.0634 + ], + "mean_e2el_ms": [ + 25070.9523, + 31992.0646, + 37619.7669, + 50393.8257, + 136619.2743 + ], + "mean_audio_rtf": [ + 0.169, + 0.2111, + 0.2475, + 0.3551, + 0.9266 + ] + } } }, { @@ -90,15 +93,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 343.1377 - ], - "mean_e2el_ms": [ - 4811.1123 - ], - "mean_audio_rtf": [ - 0.1463 - ] + "H100": { + "mean_ttft_ms": [ + 343.1377 + ], + "mean_e2el_ms": [ + 4811.1123 + ], + "mean_audio_rtf": [ + 0.1463 + ] + } } }, { @@ -127,15 +132,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 245.5446 - ], - "mean_e2el_ms": [ - 5019.1256 - ], - "mean_audio_rtf": [ - 0.1713 - ] + "H100": { + "mean_ttft_ms": [ + 245.5446 + ], + "mean_e2el_ms": [ + 5019.1256 + ], + "mean_audio_rtf": [ + 0.1713 + ] + } } }, { @@ -166,15 +173,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 506.1407 - ], - "mean_e2el_ms": [ - 6814.9656 - ], - "mean_audio_rtf": [ - 0.2213 - ] + "H100": { + "mean_ttft_ms": [ + 506.1407 + ], + "mean_e2el_ms": [ + 6814.9656 + ], + "mean_audio_rtf": [ + 0.2213 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index 4dedcd4b9..8775ee381 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -51,20 +51,22 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 69.8081, - 173.0651, - 271.8108, - 413.4079, - 497.2546 - ], - "mean_e2el_ms": [ - 4720.8225, - 7937.2532, - 9962.3566, - 13724.5479, - 18324.091 - ] + "H100": { + "mean_ttft_ms": [ + 69.8081, + 173.0651, + 271.8108, + 413.4079, + 497.2546 + ], + "mean_e2el_ms": [ + 4720.8225, + 7937.2532, + 9962.3566, + 13724.5479, + 18324.091 + ] + } } }, { @@ -97,12 +99,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 25.9477 - ], - "mean_e2el_ms": [ - 517.3155 - ] + "H100": { + "mean_ttft_ms": [ + 25.9477 + ], + "mean_e2el_ms": [ + 517.3155 + ] + } } }, { @@ -137,12 +141,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 26.7056 - ], - "mean_e2el_ms": [ - 551.308 - ] + "H100": { + "mean_ttft_ms": [ + 26.7056 + ], + "mean_e2el_ms": [ + 551.308 + ] + } } }, { @@ -179,12 +185,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 27.6906 - ], - "mean_e2el_ms": [ - 596.5284 - ] + "H100": { + "mean_ttft_ms": [ + 27.6906 + ], + "mean_e2el_ms": [ + 596.5284 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 350cfb6bb..dcd2188c5 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -34,10 +34,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0721, - "latency_mean": 13.8398, - "peak_memory_mb_max": 57610.0, - "peak_memory_mb_mean": 57610.0 + "H100": { + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 57610.0, + "peak_memory_mb_mean": 57610.0 + } } }, { @@ -52,10 +54,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0186, - "latency_mean": 53.6942, - "peak_memory_mb_max": 67790.0, - "peak_memory_mb_mean": 67790.0 + "H100": { + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 67790.0, + "peak_memory_mb_mean": 67790.0 + } } } ] @@ -66,7 +70,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } @@ -99,10 +104,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0538, - "latency_mean": 18.5591, - "peak_memory_mb_max": 56449.4286, - "peak_memory_mb_mean": 56449.4286 + "H100": { + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 56449.4286, + "peak_memory_mb_mean": 56449.4286 + } } } ] @@ -156,10 +163,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2776, - "latency_mean": 3.5938, - "peak_memory_mb_max": 58113.7143, - "peak_memory_mb_mean": 58075.1429 + "H100": { + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 58113.7143, + "peak_memory_mb_mean": 58075.1429 + } } }, { @@ -174,10 +183,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1051, - "latency_mean": 9.4918, - "peak_memory_mb_max": 68107.4286, - "peak_memory_mb_mean": 68107.4286 + "H100": { + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 68107.4286, + "peak_memory_mb_mean": 68107.4286 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 9c5ad947b..686bd6160 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -33,10 +33,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1129, - "latency_mean": 8.8352, - "peak_memory_mb_max": 58572.0, - "peak_memory_mb_mean": 58572.0 + "H100": { + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 58572.0, + "peak_memory_mb_mean": 58572.0 + } } }, { @@ -50,10 +52,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.024, - "latency_mean": 41.5686, - "peak_memory_mb_max": 66368.0, - "peak_memory_mb_mean": 66368.0 + "H100": { + "throughput_qps": 0.024, + "latency_mean": 41.5686, + "peak_memory_mb_max": 66368.0, + "peak_memory_mb_mean": 66368.0 + } } } ] @@ -96,10 +100,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0672, - "latency_mean": 14.8517, - "peak_memory_mb_max": 57076.0, - "peak_memory_mb_mean": 57076.0 + "H100": { + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 57076.0, + "peak_memory_mb_mean": 57076.0 + } } } ] @@ -152,10 +158,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2879, - "latency_mean": 3.4701, - "peak_memory_mb_max": 58758.0, - "peak_memory_mb_mean": 58758.0 + "H100": { + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 58758.0, + "peak_memory_mb_mean": 58758.0 + } } }, { @@ -169,10 +177,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1113, - "latency_mean": 8.9656, - "peak_memory_mb_max": 67424.0, - "peak_memory_mb_mean": 67424.0 + "H100": { + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 67424.0, + "peak_memory_mb_mean": 67424.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 0553d891e..bc4d93a24 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -33,10 +33,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0703, - "latency_mean": 14.208, - "peak_memory_mb_max": 63225.1429, - "peak_memory_mb_mean": 63225.1429 + "H100": { + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 + } } }, { @@ -50,10 +52,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0417, - "latency_mean": 23.914, - "peak_memory_mb_max": 63225.1429, - "peak_memory_mb_mean": 63225.1429 + "H100": { + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 3fb882563..ebf9ccc29 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -33,9 +33,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.4153, - "latency_mean": 2.4065, - "peak_memory_mb_mean": 55121.4286 + "H100": { + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 55121.4286 + } } }, { @@ -49,9 +51,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5991, - "peak_memory_mb_mean": 65118.0 + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 65118.0 + } } } ] @@ -91,9 +95,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.4175, - "latency_mean": 2.3953, - "peak_memory_mb_mean": 55088.5714 + "H100": { + "throughput_qps": 0.4175, + "latency_mean": 2.3953, + "peak_memory_mb_mean": 55088.5714 + } } }, { @@ -107,9 +113,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5768, - "peak_memory_mb_mean": 66012.2857 + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5768, + "peak_memory_mb_mean": 66012.2857 + } } } ] @@ -152,9 +160,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1214, - "latency_mean": 8.2211, - "peak_memory_mb_mean": 54501.1429 + "H100": { + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 54501.1429 + } } } ] @@ -207,9 +217,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.559, - "latency_mean": 1.794, - "peak_memory_mb_mean": 55290.5714 + "H100": { + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 55290.5714 + } } }, { @@ -223,9 +235,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1918, - "latency_mean": 5.2022, - "peak_memory_mb_mean": 64838.8571 + "H100": { + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 64838.8571 + } } } ] @@ -269,9 +283,11 @@ "warmup-num-inference-steps": 20, "enable-negative-prompt": true, "baseline": { - "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], - "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] + "H100": { + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], + "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] + } } } ] diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index cef9d087e..81035a272 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -1,6 +1,8 @@ """Tests for DFX runner metadata field exclusion.""" import json +from contextlib import contextmanager +from types import SimpleNamespace import pytest @@ -15,7 +17,7 @@ def test_task_excluded_from_cli_args(): "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "percentile-metrics": "audio_rtf,audio_ttfp", - "baseline": {"mean_audio_rtf": [0.5]}, + "baseline": {"H100": {"mean_audio_rtf": [0.5]}}, } exclude_keys = { "request_rate", @@ -121,35 +123,6 @@ def test_resolve_pytest_marks_rejects_legacy_object_format(): ) -def test_extract_mark_resource_label(): - from tests.dfx.conftest import extract_mark_resource_label - - assert extract_mark_resource_label(None) == "na" - assert ( - extract_mark_resource_label( - [ - {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, - "full_model", - ] - ) - == "H100" - ) - assert ( - extract_mark_resource_label( - [ - { - "hardware_marks": { - "res": {"cuda": "H100", "rocm": "MI325"}, - "num_cards": 2, - } - }, - "diffusion", - ] - ) - == "H100-MI325" - ) - - def test_resource_label_for_filename(): from tests.dfx.conftest import resource_label_for_filename @@ -167,16 +140,14 @@ def test_hardware_json_value(): assert hardware_json_value(None) == "" -def test_extract_configs_resource_label(monkeypatch): - from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label +def test_get_runtime_resource_label(monkeypatch): + from tests.dfx.conftest import get_runtime_resource_label monkeypatch.setattr( "tests.dfx.conftest._read_runtime_device_name", lambda *, device_id=0: "NVIDIA H100 80GB HBM3", ) - get_runtime_resource_label(refresh=True) - assert extract_configs_resource_label([]) == "H100" - get_runtime_resource_label(refresh=True) + assert get_runtime_resource_label(refresh=True) == "H100" monkeypatch.setattr( "tests.dfx.conftest._read_runtime_device_name", lambda *, device_id=0: "Ascend910B2", @@ -254,8 +225,9 @@ def test_benchmark_param_id_suffix_from_task_eval_phase(): ] -def test_create_paired_omni_benchmark_pytest_params(tmp_path): +def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_path, monkeypatch): from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params + from tests.dfx.perf.scripts import run_benchmark configs = [ { @@ -286,68 +258,230 @@ def test_create_paired_omni_benchmark_pytest_params(tmp_path): assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks) assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks) + omni_p0_server = by_id["test_omni-p0"].values[0] + omni_p1_server = by_id["test_omni-p1"].values[0] + tts_server = by_id["test_tts-p0"].values[0] + assert omni_p0_server == omni_p1_server + assert omni_p0_server != tts_server -@pytest.mark.parametrize( - ("metric", "current", "baseline"), - [ - ("request_throughput", 1.0, 1.0), - ("mean_ttft_ms", 100.0, 100.0), - ], -) -def test_omni_baseline_accepts_values_at_threshold(metric, current, baseline): - from tests.dfx.perf.scripts.run_benchmark import assert_result + events = [] - assert_result( - {"completed": 1, metric: current}, - {"baseline": {metric: baseline}}, - 1, - assert_baseline=True, - ) + @contextmanager + def fake_start(server_param): + events.append(("start", server_param)) + yield object() + events.append(("stop", server_param)) - -@pytest.mark.parametrize( - ("metric", "current", "baseline", "message"), - [ - ("request_throughput", 0.9, 1.0, "below baseline"), - ("mean_ttft_ms", 101.0, 100.0, "exceeded baseline"), - ], -) -def test_omni_baseline_rejects_regressions(metric, current, baseline, message): - from tests.dfx.perf.scripts.run_benchmark import assert_result - - with pytest.raises(AssertionError, match=message): - assert_result( - {"completed": 1, metric: current}, - {"baseline": {metric: baseline}}, - 1, - assert_baseline=True, + monkeypatch.setattr(run_benchmark, "_start_omni_server", fake_start) + active_context = run_benchmark._SingleActiveContext() + try: + first = run_benchmark.omni_server.__wrapped__( + SimpleNamespace(param=omni_p0_server), + active_context, ) + second = run_benchmark.omni_server.__wrapped__( + SimpleNamespace(param=omni_p1_server), + active_context, + ) + assert first is second + assert events == [("start", omni_p0_server)] + + third = run_benchmark.omni_server.__wrapped__( + SimpleNamespace(param=tts_server), + active_context, + ) + assert third is not first + assert events == [ + ("start", omni_p0_server), + ("stop", omni_p0_server), + ("start", tts_server), + ] + finally: + active_context.close() + + assert events == [ + ("start", omni_p0_server), + ("stop", omni_p0_server), + ("start", tts_server), + ("stop", tts_server), + ] -def test_omni_duplex_expected_audio_turns_accepts_complete_session(): - from tests.dfx.perf.scripts.run_benchmark import assert_result +def test_is_hardware_nested_baseline(): + from tests.dfx.conftest import ( + _RUNTIME_DEVICE_ALIASES, + is_hardware_nested_baseline, + ) + from tests.helpers.mark import get_hardware_mark_list - assert_result( + hardware_marks = get_hardware_mark_list() + assert hardware_marks + assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks + assert "cuda" not in hardware_marks + assert "full_model" not in hardware_marks + + # Runtime aliases are a full static list (independent of baseline allowlist). + assert "H100" in _RUNTIME_DEVICE_ALIASES + assert "A100" in _RUNTIME_DEVICE_ALIASES + + assert is_hardware_nested_baseline( { - "completed": 1, - "duplex_session_metrics": [{"audio_turn_count": 4}], - }, - {"expected_duplex_audio_turns_per_session": 4}, - 1, - assert_baseline=False, + "H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]}, + "A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]}, + } ) + # Custom metric names under a known hardware label are allowed. + assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}}) + assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}}) + # Flat metric maps / concurrency-keyed maps are not hardware-nested. + assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]}) + assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}}) + assert not is_hardware_nested_baseline({}) + # Alias-only labels (not [hardware-resource] markers) are rejected for baselines. + assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}}) + # Unknown top-level hardware label is rejected. + assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}}) + # Empty per-hardware metric map is rejected. + assert not is_hardware_nested_baseline({"H100": {}}) -def test_omni_duplex_expected_audio_turns_rejects_incomplete_session(): - from tests.dfx.perf.scripts.run_benchmark import assert_result +def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency(): + from tests.dfx.conftest import resolve_baseline_for_sweep - with pytest.raises(AssertionError, match="emitted 4 audio turns"): - assert_result( - { - "completed": 1, - "duplex_session_metrics": [{"audio_turn_count": 3}], - }, - {"expected_duplex_audio_turns_per_session": 4}, - 1, - assert_baseline=False, + baseline = { + "H100": { + "mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8], + "mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0], + }, + "A3": { + "mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0], + "mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0], + }, + } + # max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32 + got = resolve_baseline_for_sweep(baseline, sweep_index=4) + assert got == { + "H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0}, + "A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0}, + } + # First sweep step keeps both hardware buckets too. + got0 = resolve_baseline_for_sweep(baseline, sweep_index=0) + assert set(got0) == {"H100", "A3"} + assert got0["H100"]["mean_ttft_ms"] == 96.4 + assert got0["A3"]["mean_ttft_ms"] == 90.0 + + +def test_resolve_baseline_for_sweep_rejects_flat_baseline(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"): + resolve_baseline_for_sweep( + {"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]}, + sweep_index=1, ) + + +def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"): + resolve_baseline_for_sweep( + {"A100": {"throughput_qps": 1.0}}, + sweep_index=0, + ) + + +def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware(): + from tests.dfx.conftest import resolve_baseline_for_sweep + + # Sweep-aligned lists under each hardware bucket (canonical form). + listed = { + "H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]}, + "A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]}, + } + assert resolve_baseline_for_sweep(listed, sweep_index=1) == { + "H100": {"throughput_qps": 0.6, "latency_mean": 2.0}, + "A3": {"throughput_qps": 0.7, "latency_mean": 1.8}, + } + + # Scalars under hardware stay as-is (single-concurrency cases). + scalar = {"H100": {"throughput_qps": 0.5}} + assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}} + # Custom metric names are preserved. + custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}} + assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}} + assert resolve_baseline_for_sweep(None) == {} + assert resolve_baseline_for_sweep({}) == {} + + +def test_resolve_baseline_value_errors(): + from tests.dfx.conftest import resolve_baseline_value + + with pytest.raises(ValueError, match="sweep_index"): + resolve_baseline_value([1.0, 2.0], sweep_index=None) + with pytest.raises(IndexError): + resolve_baseline_value([1.0], sweep_index=1) + with pytest.raises(TypeError, match="not supported"): + resolve_baseline_value({"1": 0.4}, sweep_index=0) + + +def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch): + """``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency.""" + import importlib + import sys + + cfg = tmp_path / "mini_diffusion_perf.json" + cfg.write_text( + json.dumps( + [ + { + "test_name": "test_mini", + "server_type": "vllm-omni", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "diffusion", + "full_model", + ], + "server_params": {"model": "m/mini"}, + "benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}], + } + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)]) + sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None) + from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb + + importlib.reload(rdb) + + params = { + "name": "c_sweep", + "dataset": "random", + "task": "t2i", + "num-prompts": [8, 16, 32], + "max-concurrency": [1, 8, 32], + "baseline": { + "H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]}, + "A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]}, + }, + } + run32 = rdb._build_run_params( + params, + num_prompts=32, + max_concurrency=32, + request_rate="inf", + sweep_index=2, + ) + assert run32["max-concurrency"] == 32 + assert run32["baseline"] == { + "H100": {"throughput_qps": 0.3, "latency_mean": 30.0}, + "A3": {"throughput_qps": 0.33, "latency_mean": 29.0}, + } + + sweeps = rdb._iter_sweep_runs(params) + assert len(sweeps) == 3 + assert sweeps[2]["params"]["max-concurrency"] == 32 + assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3 + assert "A3" in sweeps[2]["params"]["baseline"] + assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 91037aab6..f398e3271 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -39,12 +39,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 141.2791 - ], - "median_audio_rtf": [ - 0.1591 - ] + "H100": { + "median_audio_ttfp_ms": [ + 141.2791 + ], + "median_audio_rtf": [ + 0.1591 + ] + } } }, { @@ -67,21 +69,23 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 226.747, - 647.1272, - 4777.2714 - ], - "median_audio_rtf": [ - 0.3313, - 0.3642, - 1.2282 - ], - "audio_throughput": [ - 24.2296, - 40.3552, - 43.2572 - ] + "H100": { + "median_audio_ttfp_ms": [ + 226.747, + 647.1272, + 4777.2714 + ], + "median_audio_rtf": [ + 0.3313, + 0.3642, + 1.2282 + ], + "audio_throughput": [ + 24.2296, + 40.3552, + 43.2572 + ] + } } }, { @@ -102,9 +106,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.45 - ] + "H100": { + "mean_audio_rtf": [ + 0.45 + ] + } } } ] @@ -153,12 +159,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 46.8434 - ], - "median_audio_rtf": [ - 0.1367 - ] + "H100": { + "median_audio_ttfp_ms": [ + 46.8434 + ], + "median_audio_rtf": [ + 0.1367 + ] + } } }, { @@ -186,21 +194,23 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 75.3341, - 713.7727, - 5941.7179 - ], - "median_audio_rtf": [ - 0.1859, - 0.3148, - 1.1662 - ], - "audio_throughput": [ - 38.3858, - 55.3523, - 68.7032 - ] + "H100": { + "median_audio_ttfp_ms": [ + 75.3341, + 713.7727, + 5941.7179 + ], + "median_audio_rtf": [ + 0.1859, + 0.3148, + 1.1662 + ], + "audio_throughput": [ + 38.3858, + 55.3523, + 68.7032 + ] + } } }, { @@ -226,9 +236,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.35 - ] + "H100": { + "mean_audio_rtf": [ + 0.35 + ] + } } }, { @@ -252,15 +264,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 56.746 - ], - "median_audio_rtf": [ - 0.1571 - ], - "audio_throughput": [ - 11.1226 - ] + "H100": { + "median_audio_ttfp_ms": [ + 56.746 + ], + "median_audio_rtf": [ + 0.1571 + ], + "audio_throughput": [ + 11.1226 + ] + } } }, { @@ -283,12 +297,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 46.3479 - ], - "median_audio_rtf": [ - 0.1352 - ] + "H100": { + "median_audio_ttfp_ms": [ + 46.3479 + ], + "median_audio_rtf": [ + 0.1352 + ] + } } }, { @@ -315,21 +331,23 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 75.5967, - 587.5877, - 4960.3726 - ], - "median_audio_rtf": [ - 0.1871, - 0.3143, - 1.1703 - ], - "audio_throughput": [ - 38.9378, - 55.1846, - 71.5708 - ] + "H100": { + "median_audio_ttfp_ms": [ + 75.5967, + 587.5877, + 4960.3726 + ], + "median_audio_rtf": [ + 0.1871, + 0.3143, + 1.1703 + ], + "audio_throughput": [ + 38.9378, + 55.1846, + 71.5708 + ] + } } }, { @@ -352,15 +370,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 55.5029 - ], - "median_audio_rtf": [ - 0.1508 - ], - "audio_throughput": [ - 9.4885 - ] + "H100": { + "median_audio_ttfp_ms": [ + 55.5029 + ], + "median_audio_rtf": [ + 0.1508 + ], + "audio_throughput": [ + 9.4885 + ] + } } } ] @@ -402,12 +422,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 82.1886 - ], - "median_audio_rtf": [ - 0.1241 - ] + "H100": { + "median_audio_ttfp_ms": [ + 82.1886 + ], + "median_audio_rtf": [ + 0.1241 + ] + } } }, { @@ -429,15 +451,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 160.9168 - ], - "median_audio_rtf": [ - 0.1907 - ], - "audio_throughput": [ - 39.2332 - ] + "H100": { + "median_audio_ttfp_ms": [ + 160.9168 + ], + "median_audio_rtf": [ + 0.1907 + ], + "audio_throughput": [ + 39.2332 + ] + } } }, { @@ -459,15 +483,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 100.1265 - ], - "median_audio_rtf": [ - 0.1437 - ], - "audio_throughput": [ - 11.7605 - ] + "H100": { + "median_audio_ttfp_ms": [ + 100.1265 + ], + "median_audio_rtf": [ + 0.1437 + ], + "audio_throughput": [ + 11.7605 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index 12afdc132..e5163e5bb 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -37,12 +37,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 137.645 - ], - "median_audio_rtf": [ - 0.0497 - ] + "H100": { + "median_audio_ttfp_ms": [ + 137.645 + ], + "median_audio_rtf": [ + 0.0497 + ] + } } }, { @@ -61,15 +63,17 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 475.3357 - ], - "median_audio_rtf": [ - 0.1243 - ], - "audio_throughput": [ - 36.6358 - ] + "H100": { + "median_audio_ttfp_ms": [ + 475.3357 + ], + "median_audio_rtf": [ + 0.1243 + ], + "audio_throughput": [ + 36.6358 + ] + } } }, { @@ -90,9 +94,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.45 - ] + "H100": { + "mean_audio_rtf": [ + 0.45 + ] + } } }, { @@ -114,12 +120,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 79.6927 - ], - "median_audio_rtf": [ - 0.0423 - ] + "H100": { + "median_audio_ttfp_ms": [ + 79.6927 + ], + "median_audio_rtf": [ + 0.0423 + ] + } } }, { @@ -141,15 +149,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 262.6212 - ], - "median_audio_rtf": [ - 0.0753 - ], - "audio_throughput": [ - 103.5843 - ] + "H100": { + "median_audio_ttfp_ms": [ + 262.6212 + ], + "median_audio_rtf": [ + 0.0753 + ], + "audio_throughput": [ + 103.5843 + ] + } } }, { @@ -171,15 +181,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 80.1095 - ], - "median_audio_rtf": [ - 0.0479 - ], - "audio_throughput": [ - 17.2918 - ] + "H100": { + "median_audio_ttfp_ms": [ + 80.1095 + ], + "median_audio_rtf": [ + 0.0479 + ], + "audio_throughput": [ + 17.2918 + ] + } } }, { @@ -203,9 +215,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.35 - ] + "H100": { + "mean_audio_rtf": [ + 0.35 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 53c7da9b7..5cea0c013 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -42,9 +42,11 @@ } ], "baseline": { - "throughput_qps": 0.0434, - "latency_mean": 23.0654, - "peak_memory_mb_mean": 76360.0 + "H100": { + "throughput_qps": 0.0434, + "latency_mean": 23.0654, + "peak_memory_mb_mean": 76360.0 + } } } ] @@ -55,7 +57,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } @@ -96,9 +99,11 @@ } ], "baseline": { - "throughput_qps": 0.0606, - "latency_mean": 16.4928, - "peak_memory_mb_mean": 47164.8 + "H100": { + "throughput_qps": 0.0606, + "latency_mean": 16.4928, + "peak_memory_mb_mean": 47164.8 + } } }, { @@ -121,9 +126,11 @@ } ], "baseline": { - "throughput_qps": 0.0124, - "latency_mean": 80.7784, - "peak_memory_mb_mean": 55374.3429 + "H100": { + "throughput_qps": 0.0124, + "latency_mean": 80.7784, + "peak_memory_mb_mean": 55374.3429 + } } } ] diff --git a/tests/helpers/mark.py b/tests/helpers/mark.py index 3475a0012..676732a8e 100644 --- a/tests/helpers/mark.py +++ b/tests/helpers/mark.py @@ -2,10 +2,69 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).""" +from __future__ import annotations + +import re +from functools import lru_cache +from pathlib import Path + import pytest from vllm.platforms import current_platform -# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor). +# Marker description tag in ``pyproject.toml`` ``tool.pytest.ini_options.markers``. +# Example: ``"H100: [hardware-resource] Tests that require H100 GPU"``. +_HARDWARE_RESOURCE_MARKER_TAG = "[hardware-resource]" + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _load_pyproject() -> dict: + path = _repo_root() / "pyproject.toml" + try: + try: + import tomllib + except ModuleNotFoundError: # Python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + + with path.open("rb") as f: + return tomllib.load(f) + except Exception: + return {} + + +@lru_cache(maxsize=1) +def get_hardware_mark_list() -> frozenset[str]: + """Return hardware SKU marker names from ``pyproject.toml``. + + A marker is treated as a hardware resource marker when its description + contains the ``[hardware-resource]`` tag, e.g.:: + + "H100: [hardware-resource] Tests that require H100 GPU" + """ + data = _load_pyproject() + entries = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("markers", []) + if not entries: + # Fallback when TOML parsing fails: scan tagged marker lines. + text = (_repo_root() / "pyproject.toml").read_text(encoding="utf-8") + return frozenset( + re.findall( + rf'^\s*"([A-Za-z0-9_]+)\s*:\s*{re.escape(_HARDWARE_RESOURCE_MARKER_TAG)}', + text, + flags=re.M, + ) + ) + + names: set[str] = set() + for entry in entries: + text = str(entry) + if _HARDWARE_RESOURCE_MARKER_TAG not in text: + continue + name = text.split(":", 1)[0].strip() + if name: + names.add(name) + return frozenset(names) def cuda_marks(*, res: str, num_cards: int):