From 7fb62db5c226558162dbee5b204d8d57479d28a0 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:35:41 +0800 Subject: [PATCH] [CI]For NPU CI, Nest perf baselines by hardware label (H100/A3) (#5402) Signed-off-by: wangyu <410167048@qq.com> --- .../test_examples/l4_performance_tests.inc.md | 14 +- recipes/Tencent/HunyuanImage-3.0-Instruct.md | 9 +- tests/dfx/conftest.py | 154 +---------- tests/dfx/perf/scripts/run_benchmark.py | 66 +---- .../perf/scripts/run_diffusion_benchmark.py | 115 +------- .../dfx/perf/tests/test_bagel_vllm_omni.json | 40 +-- .../test_boogu_image_edit_vllm_omni.json | 10 +- .../tests/test_boogu_image_vllm_omni.json | 10 +- .../perf/tests/test_cosmos3_vllm_omni.json | 32 ++- tests/dfx/perf/tests/test_higgs_audio_v3.json | 34 +-- .../perf/tests/test_hunyuan_image3_it2i.json | 13 +- .../tests/test_hunyuan_image_tp2_cfgp2.json | 11 +- .../tests/test_hunyuan_image_tp2_sp2.json | 11 +- .../perf/tests/test_hunyuan_image_tp4.json | 13 +- .../tests/test_lingbot_video_vllm_omni.json | 3 +- .../tests/test_qwen3_omni_async_chunk.json | 176 +++++++------ .../tests/test_qwen3_omni_multi_replicas.json | 34 ++- .../tests/test_qwen3_omni_no_async_chunk.json | 104 ++++---- .../perf/tests/test_qwen3_omni_vllm_text.json | 72 ++--- .../test_qwen_image_edit_2511_vllm_omni.json | 50 ++-- .../tests/test_qwen_image_edit_vllm_omni.json | 50 ++-- .../test_qwen_image_layered_vllm_omni.json | 20 +- .../perf/tests/test_qwen_image_vllm_omni.json | 64 +++-- tests/dfx/perf/tests/test_tts.json | 248 ++++++++++-------- tests/dfx/perf/tests/test_voxcpm2.json | 104 ++++---- .../perf/tests/test_wan22_i2v_vllm_omni.json | 24 +- 26 files changed, 675 insertions(+), 806 deletions(-) diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index 9bdcd29fe..5595732de 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -44,9 +44,11 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [500, 800], - "mean_audio_ttfp_ms": [2000, 3500], - "mean_audio_rtf": [0.25, 0.35] + "H100": { + "mean_ttft_ms": [500, 800], + "mean_audio_ttfp_ms": [2000, 3500], + "mean_audio_rtf": [0.25, 0.35] + } } } ] @@ -142,10 +144,6 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json - -# Optional baseline assertion (default off) -pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \ - --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json ``` See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). @@ -202,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par | num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | | request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | | max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | -| baseline | object | No | see above | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string). +| baseline | object | No | see below | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS. diff --git a/recipes/Tencent/HunyuanImage-3.0-Instruct.md b/recipes/Tencent/HunyuanImage-3.0-Instruct.md index f169e6a6b..a9a63d5ce 100644 --- a/recipes/Tencent/HunyuanImage-3.0-Instruct.md +++ b/recipes/Tencent/HunyuanImage-3.0-Instruct.md @@ -164,10 +164,11 @@ curl -s http://localhost:8091/v1/chat/completions \ PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds performance CI configs for the same DiT-only settings. The CI step is currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail` -enabled, intended for initial data collection. Performance assertions are -skipped (`skip-performance-assertion: true`); the baseline values in the -JSON configs are reference-only and will be promoted to regression gates -once enough nightly data has been collected. +enabled, intended for initial data collection. Perf runners no longer +compare metrics against JSON baselines at assert time; the `baseline` +blocks in the configs are reference-only and are copied into result JSON +for reporting. Regression gates can be reintroduced once enough nightly +data has been collected. The user-facing equivalent is to launch one of the CLI commands above and generate 1024x1024 images with 50 denoising steps. diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 2eb88bbd6..dbd6cc2d0 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -1,16 +1,15 @@ +import copy import json import os import re import subprocess import threading -import time from datetime import datetime from pathlib import Path from typing import Any import pytest -from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config @@ -119,32 +118,6 @@ def create_unique_server_pytest_params( ] -def create_benchmark_pytest_params( - benchmark_configs: list[dict[str, Any]], - server_to_benchmark_mapping: dict[str, dict], -) -> list[Any]: - """Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks.""" - marks_by_name = _marks_by_test_name(benchmark_configs) - params: list[Any] = [] - seen: set[str] = set() - for config in benchmark_configs: - test_name = config["test_name"] - if test_name in seen: - continue - seen.add(test_name) - params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) - id_suffixes = _unique_benchmark_param_id_suffixes(params_list) - for idx, id_suffix in enumerate(id_suffixes): - params.append( - pytest.param( - (test_name, idx), - marks=marks_by_name.get(test_name, []), - id=f"{test_name}-{id_suffix}", - ) - ) - return params - - def create_paired_benchmark_pytest_params( server_entries: list[tuple[Any, str]], params_by_test_name: dict[str, list[Any]], @@ -357,10 +330,6 @@ def supports_video_generation(model_name: str) -> bool: return any(key in lower for key in ("wan", "video", "i2v", "t2v")) -def supports_chat_generation(model_name: str) -> bool: - return not supports_video_generation(model_name) - - def parse_stage_devices(stage_config_path: str) -> str: text = Path(stage_config_path).read_text(encoding="utf-8") raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE) @@ -389,48 +358,6 @@ def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> N assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}" -def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None: - """Poll a minimal chat request until success.""" - deadline = time.time() + timeout_sec - payload = json.dumps( - { - "model": model, - "messages": [{"role": "user", "content": "Say hello in one short sentence."}], - "stream": False, - "modalities": ["text"], - } - ) - last_error: str | None = None - while time.time() < deadline: - try: - status, body = post_chat_completions_raw(host, port, payload) - if status == 200: - return - last_error = f"http={status} body={body[:200]!r}" - except Exception as exc: # noqa: BLE001 - last_error = str(exc) - time.sleep(2) - raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}") - - -def assert_no_extra_worker_processes( - baseline_pids: set[int], - worker_pattern: str, - timeout_sec: int = 60, -) -> None: - """Ensure no extra worker PIDs remain after teardown.""" - deadline = time.time() + timeout_sec - while time.time() < deadline: - current = set(list_remote_process_pids_by_pattern(worker_pattern)) - extra = current - baseline_pids - if not extra: - return - time.sleep(2) - current = set(list_remote_process_pids_by_pattern(worker_pattern)) - extra = sorted(current - baseline_pids) - assert not extra, f"orphan worker processes remain after container teardown: {extra}" - - def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]: mapping = {} for config in configs: @@ -637,57 +564,6 @@ def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str: return get_runtime_resource_label() -def resolve_baseline_value( - baseline_raw: Any, - *, - sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> Any: - """Pick the baseline threshold for this sweep step.""" - if baseline_raw is None: - return 100000 - if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" - ) - if isinstance(baseline_raw, (list, tuple)): - if sweep_index is None: - raise ValueError("list baseline requires sweep_index") - if not (0 <= sweep_index < len(baseline_raw)): - raise IndexError(f"baseline list len={len(baseline_raw)} has no index {sweep_index}") - return baseline_raw[sweep_index] - return baseline_raw - - -def _baseline_thresholds_for_step( - baseline_data: dict[str, Any], - *, - sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> dict[str, Any]: - """Resolve baseline config to one threshold per metric for this iteration.""" - return { - metric_name: resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - for metric_name, baseline_raw in baseline_data.items() - } - - def run_benchmark( args: list[str], test_name: str, @@ -696,18 +572,17 @@ def run_benchmark( num_prompt: int, *, baseline_config: dict[str, Any] | None = None, - sweep_index: int | None = None, - request_rate: Any | None = None, - max_concurrency: Any | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, resource_label: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. - After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds - (when ``baseline_config`` is provided). If the benchmark exits without writing a result file, - ``result_omni_template.json`` is used as a fallback. + After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the raw + ``baseline`` object from the test config (when ``baseline_config`` is provided), + including any hardware-nested map (e.g. ``{"H100": {...}}``). Downstream report + tooling can select a hardware bucket when needed. If the benchmark exits without + writing a result file, ``result_omni_template.json`` is used as a fallback. """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) @@ -774,12 +649,8 @@ def run_benchmark( result = json.load(f) if baseline_config: - result["baseline"] = _baseline_thresholds_for_step( - baseline_config, - sweep_index=sweep_index, - request_rate=request_rate, - max_concurrency=max_concurrency, - ) + # Persist the JSON baseline as-is (no hardware / sweep resolution). + result["baseline"] = copy.deepcopy(baseline_config) else: result["baseline"] = {} if random_input_len is not None: @@ -800,12 +671,3 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"), ) - parser.addoption( - "--assert-baseline", - action="store_true", - default=False, - help=( - "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config " - "(default: off)." - ), - ) diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index f37ac6ada..2c414a961 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -15,15 +15,10 @@ from tests.dfx.conftest import ( get_runtime_resource_label, is_diffusion_perf_config, load_benchmark_configs, - resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer -# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` -# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are -# defined in the same module. -# # Optional JSON field ``mark`` is applied as pytest marks via # ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks": # {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``). @@ -162,36 +157,8 @@ def benchmark_params(request): } -def assert_result( - result, - params, - num_prompt, - *, - assert_baseline: bool, - sweep_index: int | None = None, - max_concurrency: Any | None = None, - request_rate: Any | None = None, -) -> None: +def assert_result(result, num_prompt) -> None: assert result["completed"] == num_prompt, "Request failures exist" - if not assert_baseline: - return - baseline_data = params.get("baseline", {}) - for metric_name, baseline_raw in baseline_data.items(): - current_value = result[metric_name] - baseline_value = resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - if "throughput" in metric_name: - if current_value <= baseline_value: - print( - f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}" - ) - else: - if current_value >= baseline_value: - print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}") @pytest.mark.benchmark @@ -200,7 +167,7 @@ def assert_result( paired_benchmark_params, indirect=["omni_server", "benchmark_params"], ) -def test_performance_benchmark(omni_server, benchmark_params, request): +def test_performance_benchmark(omni_server, benchmark_params): test_name = benchmark_params["test_name"] params = benchmark_params["params"] dataset_name = params.get("dataset_name", "") @@ -212,7 +179,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request): print(f"Running benchmark for model: {model}") print(f"Benchmark parameters: {benchmark_params}") - assert_baseline = request.config.getoption("--assert-baseline", default=False) resource_label = get_runtime_resource_label() def to_list(value, default=None): @@ -271,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): break # QPS / request-rate sweep - for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): + for qps, num_prompt in zip(qps_list, num_prompt_list): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] result = run_benchmark( args=args, @@ -280,24 +246,14 @@ def test_performance_benchmark(omni_server, benchmark_params, request): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), - sweep_index=i, - request_rate=qps, - max_concurrency=None, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, ) - assert_result( - result, - params, - num_prompt, - assert_baseline=assert_baseline, - sweep_index=i, - request_rate=qps, - ) + assert_result(result, num_prompt) # concurrency test - for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): + for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list): args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] result = run_benchmark( args=args, @@ -306,18 +262,8 @@ def test_performance_benchmark(omni_server, benchmark_params, request): dataset_name=dataset_name, num_prompt=num_prompt, baseline_config=params.get("baseline"), - sweep_index=i, - request_rate=None, - max_concurrency=concurrency, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), resource_label=resource_label, ) - assert_result( - result, - params, - num_prompt, - assert_baseline=assert_baseline, - sweep_index=i, - max_concurrency=concurrency, - ) + assert_result(result, num_prompt) diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 375d2a440..43aff27f5 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -13,8 +13,6 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j pytest run_diffusion_benchmark.py -m "diffusion" pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). - Optional JSON field ``mark`` is applied as pytest marks on that case via ``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``). @@ -25,6 +23,7 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u the same per-file aggregation; ``-m`` only selects which cases run. """ +import copy import json import os import socket @@ -274,11 +273,6 @@ def _write_result_record(record: dict[str, Any]) -> Path: return target -def _append_to_aggregated_file(record: dict[str, Any]) -> None: - """Backward-compatible wrapper; prefer :func:`_write_result_record`.""" - _write_result_record(record) - - _server_lock = threading.Lock() # --------------------------------------------------------------------------- @@ -678,7 +672,7 @@ def run_benchmark( with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) - exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"} + exclude_keys = {"baseline", "dataset", "task", "name"} cmd = [ sys.executable, @@ -772,12 +766,17 @@ def run_benchmark( completed = metrics.get("completed_requests", metrics.get("completed", 0)) failed = metrics.get("failed_requests", metrics.get("failed", 0)) + # Persist the JSON baseline as-is (no hardware / sweep resolution). + baseline = copy.deepcopy(params.get("baseline") or {}) + metrics["baseline"] = baseline + record: dict[str, Any] = { "test_name": test_name, "endpoint": endpoint, "timestamp": timestamp, "server_params": server_cfg.get("server_params"), "benchmark_params": params, + "baseline": baseline, "result": metrics, "log_file": str(log_file), "Model": model, @@ -827,70 +826,14 @@ def run_benchmark( # --------------------------------------------------------------------------- -def _resolve_baseline_value( - baseline_raw: Any, - *, - sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> Any: - """Pick the baseline threshold for this sweep step.""" - if baseline_raw is None: - return 100000 - if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" - ) - if isinstance(baseline_raw, (list, tuple)): - if sweep_index is None: - raise ValueError("sweep_index is required when baseline is a list or tuple") - return baseline_raw[sweep_index] - return baseline_raw - - def assert_result( result: dict[str, Any], - params: dict[str, Any], num_prompts: int, - *, - sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, - assert_baseline: bool = True, ) -> None: - """Assert that benchmark metrics satisfy the configured baselines.""" + """Assert that the expected number of requests completed.""" completed = result.get("completed_requests", result.get("completed", 0)) assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" - if not assert_baseline: - return - if params.get("skip-performance-assertion", False): - print("Skipping performance assertions.") - return - - for metric, baseline_raw in params.get("baseline", {}).items(): - current = result.get(metric) - assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}" - threshold = _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - if "throughput" in metric: - assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}" - else: - assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}" - def _default_benchmark_endpoint_for_task(task: str) -> str: """Return the default client-side benchmark endpoint for a diffusion task.""" @@ -919,7 +862,6 @@ def _build_run_params( params: dict[str, Any], *, num_prompts: int, - sweep_index: int | None = None, request_rate: Any | None = None, max_concurrency: Any | None = None, ) -> dict[str, Any]: @@ -931,16 +873,6 @@ def _build_run_params( run_params["request-rate"] = request_rate if max_concurrency is not None: run_params["max-concurrency"] = max_concurrency - if "baseline" in params: - run_params["baseline"] = { - metric: _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - for metric, baseline_raw in params["baseline"].items() - } return run_params @@ -963,36 +895,28 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: sweep_runs: list[dict[str, Any]] = [] - for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)): + for request_rate, num_prompts in zip(request_rate_list, num_prompt_list): sweep_runs.append( { "params": _build_run_params( params, request_rate=request_rate, num_prompts=num_prompts, - sweep_index=i, ), "num_prompts": num_prompts, - "sweep_index": i, - "request_rate": request_rate, - "max_concurrency": None, } ) - for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)): + for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list): sweep_runs.append( { "params": _build_run_params( params, max_concurrency=max_concurrency, num_prompts=num_prompts, - sweep_index=i, request_rate="inf", ), "num_prompts": num_prompts, - "sweep_index": i, - "request_rate": None, - "max_concurrency": max_concurrency, } ) @@ -1005,9 +929,6 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: num_prompts=default_num_prompts, ), "num_prompts": default_num_prompts, - "sweep_index": None, - "request_rate": None, - "max_concurrency": None, } ) @@ -1023,14 +944,12 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: paired_benchmark_params, indirect=["diffusion_server", "benchmark_params"], ) -def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): +def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): """Run the diffusion performance benchmark and verify request completion. One server is started per unique parallel configuration (module scope). For each server, all benchmark parameter sets defined in the config JSON are executed sequentially; metrics are recorded to the aggregated result file. - - Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block. """ test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -1073,14 +992,4 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}") print("=" * 60) - assert_baseline = request.config.getoption("--assert-baseline", default=False) - - assert_result( - result, - params, - sweep_run["num_prompts"], - sweep_index=sweep_run["sweep_index"], - max_concurrency=sweep_run["max_concurrency"], - request_rate=sweep_run["request_rate"], - assert_baseline=assert_baseline, - ) + assert_result(result, sweep_run["num_prompts"]) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index e6777acfe..9f6ff0241 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -40,10 +40,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.495, - "latency_mean": 2.0238, - "peak_memory_mb_max": 29393.4286, - "peak_memory_mb_mean": 29392.7143 + "H100": { + "throughput_qps": 0.495, + "latency_mean": 2.0238, + "peak_memory_mb_max": 29393.4286, + "peak_memory_mb_mean": 29392.7143 + } } } ] @@ -90,10 +92,12 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.3451, - "latency_mean": 2.9031, - "peak_memory_mb_max": 30480.2857, - "peak_memory_mb_mean": 30479.2857 + "H100": { + "throughput_qps": 0.3451, + "latency_mean": 2.9031, + "peak_memory_mb_max": 30480.2857, + "peak_memory_mb_mean": 30479.2857 + } } } ] @@ -139,10 +143,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.5048, - "latency_mean": 1.987, - "peak_memory_mb_max": 29380.0, - "peak_memory_mb_mean": 29380.0 + "H100": { + "throughput_qps": 0.5048, + "latency_mean": 1.987, + "peak_memory_mb_max": 29380.0, + "peak_memory_mb_mean": 29380.0 + } } } ] @@ -189,10 +195,12 @@ "enable-negative-prompt": true, "num-input-images": 1, "baseline": { - "throughput_qps": 0.149, - "latency_mean": 6.7157, - "peak_memory_mb_max": 30574.0, - "peak_memory_mb_mean": 30574.0 + "H100": { + "throughput_qps": 0.149, + "latency_mean": 6.7157, + "peak_memory_mb_max": 30574.0, + "peak_memory_mb_mean": 30574.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json index 028889af1..21f2caf87 100644 --- a/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json @@ -24,10 +24,12 @@ "warmup-num-inference-steps": 28, "enable-negative-prompt": true, "baseline": { - "throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302}, - "latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7}, - "peak_memory_mb_max": 40500, - "peak_memory_mb_mean": 40500 + "H100": { + "throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302}, + "latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7}, + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } } } ] diff --git a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json index 4ee6ca577..3df89290f 100644 --- a/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_boogu_image_vllm_omni.json @@ -24,10 +24,12 @@ "warmup-num-inference-steps": 28, "enable-negative-prompt": true, "baseline": { - "throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549}, - "latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4}, - "peak_memory_mb_max": 40500, - "peak_memory_mb_mean": 40500 + "H100": { + "throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549}, + "latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4}, + "peak_memory_mb_max": 40500, + "peak_memory_mb_mean": 40500 + } } } ] diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index 9bf31658d..c4d2305e6 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -48,9 +48,11 @@ "guardrails": false }, "baseline": { - "throughput_qps": 1.107, - "latency_mean": 0.8929, - "peak_memory_mb_mean": 80000.0 + "H100": { + "throughput_qps": 1.107, + "latency_mean": 0.8929, + "peak_memory_mb_mean": 80000.0 + } } } ] @@ -115,9 +117,11 @@ "use_duration_template": false }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.6824, - "peak_memory_mb_mean": 30155.4 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 36.6824, + "peak_memory_mb_mean": 30155.4 + } } } ] @@ -181,9 +185,11 @@ "guardrails": false }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.968, - "peak_memory_mb_mean": 30443.6 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 36.968, + "peak_memory_mb_mean": 30443.6 + } } } ] @@ -251,9 +257,11 @@ "condition_video_keep": "first" }, "baseline": { - "throughput_qps": 0.027, - "latency_mean": 37.2117, - "peak_memory_mb_mean": 30135.6 + "H100": { + "throughput_qps": 0.027, + "latency_mean": 37.2117, + "peak_memory_mb_mean": 30135.6 + } } } ] diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 04cc88ac2..75e5983b0 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -46,12 +46,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 223.3584 - ], - "median_audio_rtf": [ - 0.4393 - ] + "H100": { + "median_audio_ttfp_ms": [ + 223.3584 + ], + "median_audio_rtf": [ + 0.4393 + ] + } } }, { @@ -70,15 +72,17 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 276.575 - ], - "median_audio_rtf": [ - 0.5073 - ], - "audio_throughput": [ - 27.3728 - ] + "H100": { + "median_audio_ttfp_ms": [ + 276.575 + ], + "median_audio_rtf": [ + 0.5073 + ], + "audio_throughput": [ + 27.3728 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index d2a20af7a..33063bcd3 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -133,13 +133,14 @@ "bot-task": "think_recaption", "max-concurrency": 16, "num-prompts": 96, - "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.1874, - "latency_p99": 101.852, - "latency_mean": 79.0385, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "H100": { + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index 733579e96..76a0bb75a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -37,12 +37,13 @@ "num-inference-steps": 8, "num-prompts": 10, "max-concurrency": 1, - "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.7469, - "peak_memory_mb_max": 101100, - "peak_memory_mb_mean": 101100 + "H100": { + "throughput_qps": 0.21, + "latency_p99": 4.7469, + "peak_memory_mb_max": 101100, + "peak_memory_mb_mean": 101100 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index 27b04a469..70ed93f58 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -37,12 +37,13 @@ "num-inference-steps": 8, "num-prompts": 10, "max-concurrency": 1, - "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.20, - "latency_p99": 5.1025, - "peak_memory_mb_max": 97402, - "peak_memory_mb_mean": 97402 + "H100": { + "throughput_qps": 0.20, + "latency_p99": 5.1025, + "peak_memory_mb_max": 97402, + "peak_memory_mb_mean": 97402 + } } } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index 28842164c..6157c2f1d 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -36,13 +36,14 @@ "num-inference-steps": 8, "num-prompts": 10, "max-concurrency": 1, - "skip-performance-assertion": true, "baseline": { - "throughput_qps": 0.213, - "latency_mean": 4.6953, - "latency_p99": 4.8601, - "peak_memory_mb_max": 56912.0, - "peak_memory_mb_mean": 56912.0 + "H100": { + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json index a6751bda1..07403e78f 100644 --- a/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json +++ b/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json @@ -26,8 +26,7 @@ "warmup-requests": 1, "warmup-concurrency": 1, "warmup-num-inference-steps": 2, - "enable-negative-prompt": true, - "skip-performance-assertion": true + "enable-negative-prompt": true } ] } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index be108cbd6..a73abc5fb 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -44,27 +44,29 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 96.4061, - 140.8147, - 271.9025, - 362.2916, - 507.776 - ], - "mean_e2el_ms": [ - 18507.4582, - 28365.4956, - 31907.9613, - 48161.5829, - 72630.8601 - ], - "mean_audio_rtf": [ - 0.1175, - 0.18, - 0.2359, - 0.3278, - 0.6296 - ] + "H100": { + "mean_ttft_ms": [ + 96.4061, + 140.8147, + 271.9025, + 362.2916, + 507.776 + ], + "mean_e2el_ms": [ + 18507.4582, + 28365.4956, + 31907.9613, + 48161.5829, + 72630.8601 + ], + "mean_audio_rtf": [ + 0.1175, + 0.18, + 0.2359, + 0.3278, + 0.6296 + ] + } } }, { @@ -91,15 +93,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 354.0951 - ], - "mean_e2el_ms": [ - 3952.5615 - ], - "mean_audio_rtf": [ - 0.117 - ] + "H100": { + "mean_ttft_ms": [ + 354.0951 + ], + "mean_e2el_ms": [ + 3952.5615 + ], + "mean_audio_rtf": [ + 0.117 + ] + } } }, { @@ -128,15 +132,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 259.6317 - ], - "mean_e2el_ms": [ - 4253.3009 - ], - "mean_audio_rtf": [ - 0.1448 - ] + "H100": { + "mean_ttft_ms": [ + 259.6317 + ], + "mean_e2el_ms": [ + 4253.3009 + ], + "mean_audio_rtf": [ + 0.1448 + ] + } } }, { @@ -167,15 +173,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 457.4453 - ], - "mean_e2el_ms": [ - 5912.5623 - ], - "mean_audio_rtf": [ - 0.1967 - ] + "H100": { + "mean_ttft_ms": [ + 457.4453 + ], + "mean_e2el_ms": [ + 5912.5623 + ], + "mean_audio_rtf": [ + 0.1967 + ] + } } }, { @@ -206,20 +214,22 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 90.8033, - 202.1779, - 326.5864, - 544.8914, - 961.3929 - ], - "mean_e2el_ms": [ - 4517.3411, - 7857.9711, - 9861.1687, - 13283.3363, - 19541.0736 - ] + "H100": { + "mean_ttft_ms": [ + 90.8033, + 202.1779, + 326.5864, + 544.8914, + 961.3929 + ], + "mean_e2el_ms": [ + 4517.3411, + 7857.9711, + 9861.1687, + 13283.3363, + 19541.0736 + ] + } } }, { @@ -251,12 +261,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 318.048 - ], - "mean_e2el_ms": [ - 807.6114 - ] + "H100": { + "mean_ttft_ms": [ + 318.048 + ], + "mean_e2el_ms": [ + 807.6114 + ] + } } }, { @@ -290,12 +302,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 186.168 - ], - "mean_e2el_ms": [ - 726.1137 - ] + "H100": { + "mean_ttft_ms": [ + 186.168 + ], + "mean_e2el_ms": [ + 726.1137 + ] + } } }, { @@ -331,12 +345,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 375.5282 - ], - "mean_e2el_ms": [ - 1006.1663 - ] + "H100": { + "mean_ttft_ms": [ + 375.5282 + ], + "mean_e2el_ms": [ + 1006.1663 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index 4e54c6f16..a3d8aaea8 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -35,10 +35,12 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], - "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], - "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], - "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] + "H100": { + "mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085], + "mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485], + "mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275], + "mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322] + } } }, { @@ -61,9 +63,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [340.5122], - "mean_audio_ttfp_ms": [463.0752], - "mean_audio_rtf": [0.1153] + "H100": { + "mean_ttft_ms": [340.5122], + "mean_audio_ttfp_ms": [463.0752], + "mean_audio_rtf": [0.1153] + } } }, { @@ -88,9 +92,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [236.926], - "mean_audio_ttfp_ms": [395.2182], - "mean_audio_rtf": [0.1259] + "H100": { + "mean_ttft_ms": [236.926], + "mean_audio_ttfp_ms": [395.2182], + "mean_audio_rtf": [0.1259] + } } }, { @@ -117,9 +123,11 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [470.6126], - "mean_audio_ttfp_ms": [665.5474], - "mean_audio_rtf": [0.166] + "H100": { + "mean_ttft_ms": [470.6126], + "mean_audio_ttfp_ms": [665.5474], + "mean_audio_rtf": [0.166] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 3fbeafb9d..38dbfb2e3 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -44,27 +44,29 @@ "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 115.8348, - 173.1823, - 446.672, - 464.5985, - 3420.0634 - ], - "mean_e2el_ms": [ - 25070.9523, - 31992.0646, - 37619.7669, - 50393.8257, - 136619.2743 - ], - "mean_audio_rtf": [ - 0.169, - 0.2111, - 0.2475, - 0.3551, - 0.9266 - ] + "H100": { + "mean_ttft_ms": [ + 115.8348, + 173.1823, + 446.672, + 464.5985, + 3420.0634 + ], + "mean_e2el_ms": [ + 25070.9523, + 31992.0646, + 37619.7669, + 50393.8257, + 136619.2743 + ], + "mean_audio_rtf": [ + 0.169, + 0.2111, + 0.2475, + 0.3551, + 0.9266 + ] + } } }, { @@ -91,15 +93,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 343.1377 - ], - "mean_e2el_ms": [ - 4811.1123 - ], - "mean_audio_rtf": [ - 0.1463 - ] + "H100": { + "mean_ttft_ms": [ + 343.1377 + ], + "mean_e2el_ms": [ + 4811.1123 + ], + "mean_audio_rtf": [ + 0.1463 + ] + } } }, { @@ -128,15 +132,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 245.5446 - ], - "mean_e2el_ms": [ - 5019.1256 - ], - "mean_audio_rtf": [ - 0.1713 - ] + "H100": { + "mean_ttft_ms": [ + 245.5446 + ], + "mean_e2el_ms": [ + 5019.1256 + ], + "mean_audio_rtf": [ + 0.1713 + ] + } } }, { @@ -167,15 +173,17 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [ - 506.1407 - ], - "mean_e2el_ms": [ - 6814.9656 - ], - "mean_audio_rtf": [ - 0.2213 - ] + "H100": { + "mean_ttft_ms": [ + 506.1407 + ], + "mean_e2el_ms": [ + 6814.9656 + ], + "mean_audio_rtf": [ + 0.2213 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index 4dedcd4b9..8775ee381 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -51,20 +51,22 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 69.8081, - 173.0651, - 271.8108, - 413.4079, - 497.2546 - ], - "mean_e2el_ms": [ - 4720.8225, - 7937.2532, - 9962.3566, - 13724.5479, - 18324.091 - ] + "H100": { + "mean_ttft_ms": [ + 69.8081, + 173.0651, + 271.8108, + 413.4079, + 497.2546 + ], + "mean_e2el_ms": [ + 4720.8225, + 7937.2532, + 9962.3566, + 13724.5479, + 18324.091 + ] + } } }, { @@ -97,12 +99,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 25.9477 - ], - "mean_e2el_ms": [ - 517.3155 - ] + "H100": { + "mean_ttft_ms": [ + 25.9477 + ], + "mean_e2el_ms": [ + 517.3155 + ] + } } }, { @@ -137,12 +141,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 26.7056 - ], - "mean_e2el_ms": [ - 551.308 - ] + "H100": { + "mean_ttft_ms": [ + 26.7056 + ], + "mean_e2el_ms": [ + 551.308 + ] + } } }, { @@ -179,12 +185,14 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [ - 27.6906 - ], - "mean_e2el_ms": [ - 596.5284 - ] + "H100": { + "mean_ttft_ms": [ + 27.6906 + ], + "mean_e2el_ms": [ + 596.5284 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 75a3d7744..dcd2188c5 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -34,10 +34,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0721, - "latency_mean": 13.8398, - "peak_memory_mb_max": 57610.0, - "peak_memory_mb_mean": 57610.0 + "H100": { + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 57610.0, + "peak_memory_mb_mean": 57610.0 + } } }, { @@ -52,10 +54,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0186, - "latency_mean": 53.6942, - "peak_memory_mb_max": 67790.0, - "peak_memory_mb_mean": 67790.0 + "H100": { + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 67790.0, + "peak_memory_mb_mean": 67790.0 + } } } ] @@ -100,10 +104,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0538, - "latency_mean": 18.5591, - "peak_memory_mb_max": 56449.4286, - "peak_memory_mb_mean": 56449.4286 + "H100": { + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 56449.4286, + "peak_memory_mb_mean": 56449.4286 + } } } ] @@ -157,10 +163,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2776, - "latency_mean": 3.5938, - "peak_memory_mb_max": 58113.7143, - "peak_memory_mb_mean": 58075.1429 + "H100": { + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 58113.7143, + "peak_memory_mb_mean": 58075.1429 + } } }, { @@ -175,10 +183,12 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1051, - "latency_mean": 9.4918, - "peak_memory_mb_max": 68107.4286, - "peak_memory_mb_mean": 68107.4286 + "H100": { + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 68107.4286, + "peak_memory_mb_mean": 68107.4286 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 9c5ad947b..686bd6160 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -33,10 +33,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1129, - "latency_mean": 8.8352, - "peak_memory_mb_max": 58572.0, - "peak_memory_mb_mean": 58572.0 + "H100": { + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 58572.0, + "peak_memory_mb_mean": 58572.0 + } } }, { @@ -50,10 +52,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.024, - "latency_mean": 41.5686, - "peak_memory_mb_max": 66368.0, - "peak_memory_mb_mean": 66368.0 + "H100": { + "throughput_qps": 0.024, + "latency_mean": 41.5686, + "peak_memory_mb_max": 66368.0, + "peak_memory_mb_mean": 66368.0 + } } } ] @@ -96,10 +100,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0672, - "latency_mean": 14.8517, - "peak_memory_mb_max": 57076.0, - "peak_memory_mb_mean": 57076.0 + "H100": { + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 57076.0, + "peak_memory_mb_mean": 57076.0 + } } } ] @@ -152,10 +158,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.2879, - "latency_mean": 3.4701, - "peak_memory_mb_max": 58758.0, - "peak_memory_mb_mean": 58758.0 + "H100": { + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 58758.0, + "peak_memory_mb_mean": 58758.0 + } } }, { @@ -169,10 +177,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1113, - "latency_mean": 8.9656, - "peak_memory_mb_max": 67424.0, - "peak_memory_mb_mean": 67424.0 + "H100": { + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 67424.0, + "peak_memory_mb_mean": 67424.0 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 0553d891e..bc4d93a24 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -33,10 +33,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0703, - "latency_mean": 14.208, - "peak_memory_mb_max": 63225.1429, - "peak_memory_mb_mean": 63225.1429 + "H100": { + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 + } } }, { @@ -50,10 +52,12 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0417, - "latency_mean": 23.914, - "peak_memory_mb_max": 63225.1429, - "peak_memory_mb_mean": 63225.1429 + "H100": { + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 63225.1429, + "peak_memory_mb_mean": 63225.1429 + } } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 3fb882563..ebf9ccc29 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -33,9 +33,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.4153, - "latency_mean": 2.4065, - "peak_memory_mb_mean": 55121.4286 + "H100": { + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 55121.4286 + } } }, { @@ -49,9 +51,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5991, - "peak_memory_mb_mean": 65118.0 + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 65118.0 + } } } ] @@ -91,9 +95,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.4175, - "latency_mean": 2.3953, - "peak_memory_mb_mean": 55088.5714 + "H100": { + "throughput_qps": 0.4175, + "latency_mean": 2.3953, + "peak_memory_mb_mean": 55088.5714 + } } }, { @@ -107,9 +113,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5768, - "peak_memory_mb_mean": 66012.2857 + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5768, + "peak_memory_mb_mean": 66012.2857 + } } } ] @@ -152,9 +160,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1214, - "latency_mean": 8.2211, - "peak_memory_mb_mean": 54501.1429 + "H100": { + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 54501.1429 + } } } ] @@ -207,9 +217,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.559, - "latency_mean": 1.794, - "peak_memory_mb_mean": 55290.5714 + "H100": { + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 55290.5714 + } } }, { @@ -223,9 +235,11 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1918, - "latency_mean": 5.2022, - "peak_memory_mb_mean": 64838.8571 + "H100": { + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 64838.8571 + } } } ] @@ -269,9 +283,11 @@ "warmup-num-inference-steps": 20, "enable-negative-prompt": true, "baseline": { - "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], - "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] + "H100": { + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], + "peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821] + } } } ] diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 91037aab6..f398e3271 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -39,12 +39,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 141.2791 - ], - "median_audio_rtf": [ - 0.1591 - ] + "H100": { + "median_audio_ttfp_ms": [ + 141.2791 + ], + "median_audio_rtf": [ + 0.1591 + ] + } } }, { @@ -67,21 +69,23 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 226.747, - 647.1272, - 4777.2714 - ], - "median_audio_rtf": [ - 0.3313, - 0.3642, - 1.2282 - ], - "audio_throughput": [ - 24.2296, - 40.3552, - 43.2572 - ] + "H100": { + "median_audio_ttfp_ms": [ + 226.747, + 647.1272, + 4777.2714 + ], + "median_audio_rtf": [ + 0.3313, + 0.3642, + 1.2282 + ], + "audio_throughput": [ + 24.2296, + 40.3552, + 43.2572 + ] + } } }, { @@ -102,9 +106,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.45 - ] + "H100": { + "mean_audio_rtf": [ + 0.45 + ] + } } } ] @@ -153,12 +159,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 46.8434 - ], - "median_audio_rtf": [ - 0.1367 - ] + "H100": { + "median_audio_ttfp_ms": [ + 46.8434 + ], + "median_audio_rtf": [ + 0.1367 + ] + } } }, { @@ -186,21 +194,23 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 75.3341, - 713.7727, - 5941.7179 - ], - "median_audio_rtf": [ - 0.1859, - 0.3148, - 1.1662 - ], - "audio_throughput": [ - 38.3858, - 55.3523, - 68.7032 - ] + "H100": { + "median_audio_ttfp_ms": [ + 75.3341, + 713.7727, + 5941.7179 + ], + "median_audio_rtf": [ + 0.1859, + 0.3148, + 1.1662 + ], + "audio_throughput": [ + 38.3858, + 55.3523, + 68.7032 + ] + } } }, { @@ -226,9 +236,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.35 - ] + "H100": { + "mean_audio_rtf": [ + 0.35 + ] + } } }, { @@ -252,15 +264,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 56.746 - ], - "median_audio_rtf": [ - 0.1571 - ], - "audio_throughput": [ - 11.1226 - ] + "H100": { + "median_audio_ttfp_ms": [ + 56.746 + ], + "median_audio_rtf": [ + 0.1571 + ], + "audio_throughput": [ + 11.1226 + ] + } } }, { @@ -283,12 +297,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 46.3479 - ], - "median_audio_rtf": [ - 0.1352 - ] + "H100": { + "median_audio_ttfp_ms": [ + 46.3479 + ], + "median_audio_rtf": [ + 0.1352 + ] + } } }, { @@ -315,21 +331,23 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 75.5967, - 587.5877, - 4960.3726 - ], - "median_audio_rtf": [ - 0.1871, - 0.3143, - 1.1703 - ], - "audio_throughput": [ - 38.9378, - 55.1846, - 71.5708 - ] + "H100": { + "median_audio_ttfp_ms": [ + 75.5967, + 587.5877, + 4960.3726 + ], + "median_audio_rtf": [ + 0.1871, + 0.3143, + 1.1703 + ], + "audio_throughput": [ + 38.9378, + 55.1846, + 71.5708 + ] + } } }, { @@ -352,15 +370,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 55.5029 - ], - "median_audio_rtf": [ - 0.1508 - ], - "audio_throughput": [ - 9.4885 - ] + "H100": { + "median_audio_ttfp_ms": [ + 55.5029 + ], + "median_audio_rtf": [ + 0.1508 + ], + "audio_throughput": [ + 9.4885 + ] + } } } ] @@ -402,12 +422,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 82.1886 - ], - "median_audio_rtf": [ - 0.1241 - ] + "H100": { + "median_audio_ttfp_ms": [ + 82.1886 + ], + "median_audio_rtf": [ + 0.1241 + ] + } } }, { @@ -429,15 +451,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 160.9168 - ], - "median_audio_rtf": [ - 0.1907 - ], - "audio_throughput": [ - 39.2332 - ] + "H100": { + "median_audio_ttfp_ms": [ + 160.9168 + ], + "median_audio_rtf": [ + 0.1907 + ], + "audio_throughput": [ + 39.2332 + ] + } } }, { @@ -459,15 +483,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 100.1265 - ], - "median_audio_rtf": [ - 0.1437 - ], - "audio_throughput": [ - 11.7605 - ] + "H100": { + "median_audio_ttfp_ms": [ + 100.1265 + ], + "median_audio_rtf": [ + 0.1437 + ], + "audio_throughput": [ + 11.7605 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index 12afdc132..e5163e5bb 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -37,12 +37,14 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 137.645 - ], - "median_audio_rtf": [ - 0.0497 - ] + "H100": { + "median_audio_ttfp_ms": [ + 137.645 + ], + "median_audio_rtf": [ + 0.0497 + ] + } } }, { @@ -61,15 +63,17 @@ "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 475.3357 - ], - "median_audio_rtf": [ - 0.1243 - ], - "audio_throughput": [ - 36.6358 - ] + "H100": { + "median_audio_ttfp_ms": [ + 475.3357 + ], + "median_audio_rtf": [ + 0.1243 + ], + "audio_throughput": [ + 36.6358 + ] + } } }, { @@ -90,9 +94,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.45 - ] + "H100": { + "mean_audio_rtf": [ + 0.45 + ] + } } }, { @@ -114,12 +120,14 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 79.6927 - ], - "median_audio_rtf": [ - 0.0423 - ] + "H100": { + "median_audio_ttfp_ms": [ + 79.6927 + ], + "median_audio_rtf": [ + 0.0423 + ] + } } }, { @@ -141,15 +149,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 262.6212 - ], - "median_audio_rtf": [ - 0.0753 - ], - "audio_throughput": [ - 103.5843 - ] + "H100": { + "median_audio_ttfp_ms": [ + 262.6212 + ], + "median_audio_rtf": [ + 0.0753 + ], + "audio_throughput": [ + 103.5843 + ] + } } }, { @@ -171,15 +181,17 @@ }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [ - 80.1095 - ], - "median_audio_rtf": [ - 0.0479 - ], - "audio_throughput": [ - 17.2918 - ] + "H100": { + "median_audio_ttfp_ms": [ + 80.1095 + ], + "median_audio_rtf": [ + 0.0479 + ], + "audio_throughput": [ + 17.2918 + ] + } } }, { @@ -203,9 +215,11 @@ "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [ - 0.35 - ] + "H100": { + "mean_audio_rtf": [ + 0.35 + ] + } } } ] diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 52441dae4..5cea0c013 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -42,9 +42,11 @@ } ], "baseline": { - "throughput_qps": 0.0434, - "latency_mean": 23.0654, - "peak_memory_mb_mean": 76360.0 + "H100": { + "throughput_qps": 0.0434, + "latency_mean": 23.0654, + "peak_memory_mb_mean": 76360.0 + } } } ] @@ -97,9 +99,11 @@ } ], "baseline": { - "throughput_qps": 0.0606, - "latency_mean": 16.4928, - "peak_memory_mb_mean": 47164.8 + "H100": { + "throughput_qps": 0.0606, + "latency_mean": 16.4928, + "peak_memory_mb_mean": 47164.8 + } } }, { @@ -122,9 +126,11 @@ } ], "baseline": { - "throughput_qps": 0.0124, - "latency_mean": 80.7784, - "peak_memory_mb_mean": 55374.3429 + "H100": { + "throughput_qps": 0.0124, + "latency_mean": 80.7784, + "peak_memory_mb_mean": 55374.3429 + } } } ]